Fix mistakes of the old harness

This commit is contained in:
nvaldez
2026-09-20 13:15:22 +02:00
parent 895f5d0945
commit fac22f2e75
6 changed files with 267 additions and 175 deletions
+29
View File
@@ -0,0 +1,29 @@
# Fiona
Tools for managing the LLM wiki.
## Development
### Setup
1. Create and activate the virtual environment:
```bash
source bootstrap --env fiona_env -c
```
> Make sure `fiona_env` is activated as pyenv before proceeding
### Testing
Run tests with pytest:
```bash
pytest
```
### Usage
Run the fiona CLI:
```bash
fiona test
# or
python -m fiona test
```
+185 -142
View File
@@ -28,85 +28,18 @@ FORCE_PYENV_INSTALL=false # Flag to force pyenv reinstallation
CLEAN_ENVIRONMENT=false # Flag to delete and recreate environment
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # scripts root directory
source "SCRIPT_DIR/src/utils.sh"
# Parse command line arguments
while [[ $# -gt 0 ]]; do
case "$1" in
--pyenv)
FORCE_PYENV_INSTALL=true
shift
;;
-c|--clean)
CLEAN_ENVIRONMENT=true
shift
;;
--env)
export PYENV_NAME="$2"
shift 2
;;
--python)
PYTHON_VERSION="$2"
shift 2
;;
*)
echo_error "Unknown option: $1"
echo_error "Usage: source bootstrap [--pyenv] [-c|--clean] [--env name] [--python version]"
return 1
;;
esac
done
activate_environment() {
# Check if environment exists
if ! pyenv virtualenvs | grep -q "$PYENV_NAME"; then
echo_status "Creating virtual environment $PYENV_NAME..."
pyenv virtualenv "$PYTHON_VERSION" "$PYENV_NAME" || {
echo_error "Failed to create virtual environment $PYENV_NAME"
return 1
}
fi
# Activate the environment
echo_status "Activating $PYENV_NAME..."
if pyenv activate "$PYENV_NAME" 2>/dev/null; then
echo_status "Successfully activated $PYENV_NAME with pyenv"
return 0
fi
local activate_script="${SCRIPT_DIR}/src/$PYENV_NAME/bin/activate"
if [ -f "$activate_script" ]; then
source "$activate_script"
echo_status "Successfully activated $PYENV_NAME"
return 0
fi
echo_error "Failed to activate $PYENV_NAME"
return 1
}
install_requirements() {
echo_status "Installing requirements..."
# Find and install all requirements.txt files
find "${SCRIPT_DIR}/src" -name "requirements.txt" | while read -r req_file; do
echo_status "Installing requirements from: $req_file"
pip install -q -r "$req_file" --break-system-packages || {
echo_error "Failed to install requirements from $req_file"
return 1
}
done
}
# Source utilities
source "$SCRIPT_DIR/src/utils.sh"
# =============================================
# CORE FUNCTIONS
# FUNCTION DEFINITIONS
# =============================================
check_python_installed() {
# First check if Python is installed at all
if ! command -v python &> /dev/null && ! command -v python3 &> /dev/null; then
echo_error "Python is not installed. Please install Python $PYTHON_VERSION before running this script."
echo_error "On Ubuntu/Debian: sudo apt-get install python$PYTHON_VERSION"
echo_error "On Ubuntu/Debian: sudo apt-get install python3"
echo_error "On macOS: brew install python@$PYTHON_VERSION"
echo_error "Or download from: https://www.python.org/downloads/"
return 1
@@ -116,11 +49,11 @@ check_python_installed() {
if command -v pyenv &> /dev/null; then
if ! pyenv versions | grep -q "$PYTHON_VERSION"; then
echo_status "Python $PYTHON_VERSION is not available in pyenv. Installing it now..."
pyenv install "$PYTHON_VERSION" || {
if ! pyenv install "$PYTHON_VERSION"; then
echo_error "Failed to install Python $PYTHON_VERSION via pyenv"
echo_error "You may need to install it manually first"
echo_error "You may need to install build dependencies first or install it manually"
return 1
}
fi
fi
fi
@@ -133,59 +66,90 @@ initialize_pyenv() {
export PATH="$PYENV_ROOT/bin:$PATH"
# Check if pyenv is already properly initialized
if pyenv virtualenvs &> /dev/null && pyenv activate "$PYENV_NAME" &> /dev/null; then
if command -v pyenv &> /dev/null && pyenv virtualenvs &> /dev/null; then
echo_status "pyenv is already properly initialized"
return 0
fi
# Force pyenv initialization
if [ -f "$PYENV_ROOT/bin/pyenv" ]; then
echo_status "Initializing pyenv..."
# Clear any existing pyenv functions
if declare -F | grep -q 'pyenv$'; then
unset -f pyenv
fi
# Run the initialization commands directly
eval "$("$PYENV_ROOT/bin/pyenv" init -)"
eval "$("$PYENV_ROOT/bin/pyenv" virtualenv-init -)"
# Verify initialization
if pyenv virtualenvs &> /dev/null; then
echo_status "pyenv initialized successfully"
return 0
else
echo_error "Failed to initialize pyenv"
return 1
fi
else
# Check if pyenv exists
if [ ! -f "$PYENV_ROOT/bin/pyenv" ]; then
echo_error "pyenv not found at $PYENV_ROOT/bin/pyenv"
echo_error "Please install pyenv first or use --pyenv flag"
return 1
fi
echo_status "Initializing pyenv..."
# Clear any existing pyenv functions to avoid conflicts
if declare -F | grep -q 'pyenv$'; then
unset -f pyenv
fi
# Initialize pyenv and pyenv-virtualenv
eval "$("$PYENV_ROOT/bin/pyenv" init -)" || {
echo_error "Failed to initialize pyenv"
return 1
}
# Only init virtualenv plugin if available
if [ -f "$PYENV_ROOT/plugins/pyenv-virtualenv/bin/pyenv-virtualenv-init" ]; then
eval "$("$PYENV_ROOT/bin/pyenv" virtualenv-init -)" || {
echo_warning "Failed to initialize pyenv-virtualenv plugin"
}
fi
# Verify initialization
if ! pyenv virtualenvs &> /dev/null; then
echo_error "Failed to initialize pyenv"
return 1
fi
echo_status "pyenv initialized successfully"
return 0
}
install_pyenv_dependencies() {
# Install pyenv dependencies on Debian-based systems
if [ -f /etc/debian_version ]; then
if [ -f /etc/debian_version ] || [ -f /etc/os-release ] && grep -q "ID=debian\|ID=ubuntu" /etc/os-release 2>/dev/null; then
echo_status "Installing pyenv dependencies..."
if ! command -v sudo &> /dev/null; then
echo_error "sudo is required to install dependencies"
return 1
fi
sudo apt-get update && sudo apt-get install -y \
make build-essential libssl-dev zlib1g-dev \
libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \
libncurses-dev xz-utils tk-dev libffi-dev liblzma-dev \
python3-openssl git || {
echo_error "Failed to install pyenv dependencies"
echo_warning "You may need to install them manually"
return 1
}
elif [ "$(uname)" = "Darwin" ]; then
echo_status "Detected macOS - ensuring required tools are available"
if ! command -v brew &> /dev/null; then
echo_warning "Homebrew (brew) not found. You may need to install pyenv dependencies manually."
echo_warning "Recommended: brew install pyenv"
return 0
fi
if ! brew list pyenv &> /dev/null; then
echo_status "Installing pyenv via homebrew..."
brew install pyenv || {
echo_error "Failed to install pyenv via homebrew"
return 1
}
fi
fi
return 0
}
install_pyenv() {
echo_status "Installing pyenv..."
curl https://pyenv.run | bash || {
curl -fsSL https://pyenv.run | bash || {
echo_error "Failed to install pyenv"
return 1
}
return 0
}
check_and_install_pyenv() {
@@ -208,22 +172,31 @@ check_and_install_pyenv() {
fi
fi
# Install pyenv
# Install dependencies first
install_pyenv_dependencies || return 1
# Install pyenv
install_pyenv || return 1
# Source the pyenv setup in current shell
export PYENV_ROOT="$HOME/.pyenv"
export PATH="$PYENV_ROOT/bin:$PATH"
eval "$("$PYENV_ROOT/bin/pyenv" init -)"
return 0
}
clean_environment() {
if pyenv virtualenvs | grep -q "$PYENV_NAME"; then
echo_status "Deleting existing virtual environment $PYENV_NAME..."
# Redirect both stdout and stderr to /dev/null to suppress messages
pyenv uninstall -f "$PYENV_NAME" >/dev/null 2>&1 || {
echo_error "Failed to delete virtual environment $PYENV_NAME"
return 1
}
else
echo_status "Virtual environment $PYENV_NAME does not exist, skipping deletion"
fi
return 0
}
install_python_version() {
@@ -237,19 +210,17 @@ install_python_version() {
}
check_environment_exists() {
# Clean if requested
if [ "$CLEAN_ENVIRONMENT" = true ]; then
if ! clean_environment; then
return 1
fi
clean_environment || return 1
fi
# Check if Python version is available
if ! pyenv versions | grep -q "$PYTHON_VERSION"; then
if ! install_python_version "$PYTHON_VERSION"; then
return 1
fi
install_python_version "$PYTHON_VERSION" || return 1
fi
# Create environment if it doesn't exist
if ! pyenv virtualenvs | grep -q "$PYENV_NAME"; then
echo_status "Creating virtual environment $PYENV_NAME..."
pyenv virtualenv "$PYTHON_VERSION" "$PYENV_NAME" || {
@@ -266,42 +237,115 @@ check_environment_exists() {
activate_environment() {
echo_status "Activating $PYENV_NAME..."
# Try to activate the environment
if pyenv activate "$PYENV_NAME"; then
echo_status "Successfully activated $PYENV_NAME"
return 0
else
echo_error "Failed to activate $PYENV_NAME"
echo_error "Error: $(pyenv activate "$PYENV_NAME" 2>&1)"
return 1
# Try pyenv activate first
if command -v pyenv &> /dev/null; then
# Get the virtualenv path
local venv_path
venv_path=$(pyenv prefix "$PYENV_NAME" 2>/dev/null)
if [ -n "$venv_path" ] && [ -d "$venv_path" ]; then
# Manually activate by sourcing the activate script
local activate_script="$venv_path/bin/activate"
if [ -f "$activate_script" ]; then
source "$activate_script" || {
echo_error "Failed to source activation script: $activate_script"
return 1
}
echo_status "Successfully activated $PYENV_NAME"
return 0
fi
fi
# Fallback: try pyenv activate
if pyenv activate "$PYENV_NAME" 2>/dev/null; then
echo_status "Successfully activated $PYENV_NAME with pyenv"
return 0
fi
fi
}
upgrade_pip() {
echo_status "Upgrading pip..."
python -m pip install --upgrade pip || {
echo_warning "Failed to upgrade pip"
echo_warning "Continuing with requirements installation..."
}
# Last resort: check common virtualenv locations
local possible_paths=(
"$HOME/.pyenv/versions/$PYENV_NAME/bin/activate"
"$HOME/.virtualenvs/$PYENV_NAME/bin/activate"
"$SCRIPT_DIR/venv/bin/activate"
"$SCRIPT_DIR/src/venv/bin/activate"
)
for path in "${possible_paths[@]}"; do
if [ -f "$path" ]; then
source "$path" || {
echo_error "Failed to source activation script: $path"
return 1
}
echo_status "Successfully activated $PYENV_NAME"
return 0
fi
done
echo_error "Failed to activate $PYENV_NAME - could not find activation script"
return 1
}
install_requirements() {
echo_status "Installing requirements..."
# Find and install all requirements.txt files
find "${SCRIPT_DIR}/src" -name "requirements.txt" | while read -r req_file; do
local req_files=()
# Find all requirements.txt files in src directory
while IFS= read -r -d '' req_file; do
req_files+=("$req_file")
done < <(find "$SCRIPT_DIR/src" -name "requirements.txt" -print0)
if [ ${#req_files[@]} -eq 0 ]; then
echo_status "No requirements.txt files found"
return 0
fi
for req_file in "${req_files[@]}"; do
echo_status "Installing requirements from: $req_file"
pip install -r "$req_file" --break-system-packages || {
echo_error "Failed to install requirements from $req_file"
return 1
}
done
return 0
}
upgrade_pip() {
echo_status "Upgrading pip..."
python -m pip install --upgrade pip --break-system-packages || {
echo_warning "Failed to upgrade pip"
echo_warning "Continuing with requirements installation..."
}
}
install_dev_packages() {
echo_status "Installing local Python packages in development mode..."
local install_script="$SCRIPT_DIR/src/install_packages.sh"
if [ -f "$install_script" ]; then
# Use source to run in current shell context, preserving environment variables
(cd "$SCRIPT_DIR/src" && . "$install_script") || {
echo_error "Dev packages not installed!"
return 1
}
else
echo_status "Install script not found at $install_script, attempting direct install..."
# Fallback: install in editable mode from src directory
(cd "$SCRIPT_DIR/src" && pip install -e . --break-system-packages) || {
echo_error "Failed to install package in development mode"
return 1
}
fi
return 0
}
# =============================================
# MAIN EXECUTION
# =============================================
echo_status "Starting project bootstrap with environment: $PYENV_NAME"
if [ "$FORCE_PYENV_INSTALL" = true ]; then
echo_status "Forcing pyenv reinstallation"
fi
@@ -316,40 +360,39 @@ if ! check_and_install_pyenv; then
return 1
fi
# 2. Initialize pyenv
# 3. Initialize pyenv
if ! initialize_pyenv; then
return 1
fi
# 3. Check/create virtual environment
# 4. Check/create virtual environment
if ! check_environment_exists; then
return 1
fi
# 4. Activate the environment
# 5. Activate the environment
if ! activate_environment; then
return 1
fi
# 5. Upgrade pip
# 6. Upgrade pip
upgrade_pip
# 7. Install local Python packages in development mode
if [ -f "${SCRIPT_DIR}/src/install_packages.sh" ]; then
echo_status "Installing local Python packages in development mode..."
${SCRIPT_DIR}/src/install_packages.sh || {
echo_error "Dev packages not installed!"
return 1
}
else
echo_status "${SCRIPT_DIR}/src/install_packages.sh not found. Skipping local package installation."
fi
# 7. Install requirements
install_requirements
# 6. Install requirements
# install_requirements
# 8. Install local Python packages in development mode
install_dev_packages
# Final verification
echo_status "Bootstrap completed successfully!"
echo_status "Virtual environment '$PYENV_NAME' is active"
echo_status "Pyenv: $(pyenv version)"
echo_status "$(python --version) ($(which python))"
echo_status "Python: $(python --version) ($(which python))"
echo_status "Pip: $(pip --version)"
echo_status ""
echo_status "To verify installation, try:"
echo_status " python -m fiona test"
echo_status " librarian --help"
echo_status " mdoctor --help"
echo_status ""
echo_status "Happy coding!"
+15 -3
View File
@@ -6,13 +6,25 @@ It delegates commands to the appropriate sub-tools (e.g., librarian, mdoctor).
import argparse
import sys
from librarian.librarian import Librarian
from mdoctor.mdoctor import MDoctor
def main():
"""Entry point for the Fiona CLI."""
print("Fiona: Hello!")
parser = argparse.ArgumentParser(
description="Fiona - Tools for managing the LLM wiki."
)
parser.add_argument(
"command",
nargs="?",
default="hello",
help="Command to execute (e.g., 'test')"
)
args = parser.parse_args()
if args.command == "test":
print("Fiona: test")
else:
print("Fiona: Hello!")
if __name__ == "__main__":
+16 -16
View File
@@ -5,9 +5,9 @@ PIP_INSTALL_FLAGS="--force-reinstall -q"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # scripts root directory
# Default virtual environment name (can be overridden by bootstrap)
PYENV_NAME="${PYENV_NAME:-test_env}"
export PYENV_NAME="${PYENV_NAME:-test_env}"
source ${SCRIPT_DIR}/utils.sh
source "${SCRIPT_DIR}/utils.sh"
# Function to check if a package is already installed
is_installed() {
@@ -24,24 +24,24 @@ uninstall_package() {
fi
}
# Function to activate the virtual environment
# Function to check if a virtual environment is active
check_pyenv() {
# Check if pyenv is active and the correct environment is loaded
if [ -z "$PYENV_ROOT" ] || [ -z "$PYENV_VIRTUAL_ENV" ]; then
echo "Pyenv is not active or no virtual environment is loaded."
# Check if a virtual environment is active
if [ -z "$VIRTUAL_ENV" ] && [ -z "$PYENV_VIRTUAL_ENV" ]; then
echo "No virtual environment is loaded."
echo "Please activate a pyenv virtual environment first."
return 1
fi
# Extract the environment name from PYENV_VIRTUAL_ENV path
local current_env_name
current_env_name="$(basename "$PYENV_VIRTUAL_ENV")"
if [ "$current_env_name" != "$PYENV_NAME" ]; then
echo "Wrong pyenv environment: expected $PYENV_NAME, got $current_env_name"
return 1
# Get the current environment name for logging
local current_env_name=""
if [ -n "$PYENV_VIRTUAL_ENV" ]; then
current_env_name="$(basename "$PYENV_VIRTUAL_ENV")"
elif [ -n "$VIRTUAL_ENV" ]; then
current_env_name="$(basename "$VIRTUAL_ENV")"
fi
echo "Pyenv environment $PYENV_NAME is active"
echo "Pyenv environment $current_env_name is active"
return 0
}
@@ -113,6 +113,6 @@ main() {
}
# Run the main function
pushd ${SCRIPT_DIR}
pushd "${SCRIPT_DIR}" >/dev/null
main "$@"
popd
popd >/dev/null
+1 -5
View File
@@ -25,11 +25,7 @@ include = ["fiona", "librarian", "mdoctor"]
[tool.pytest.ini_options]
testpaths = ["tests"]
# Define standalone packages
[project.entry-points.console_scripts]
librarian = "librarian.librarian:main"
mdoctor = "mdoctor.mdoctor:main"
# Define console scripts entry points
[project.scripts]
librarian = "librarian.librarian:main"
mdoctor = "mdoctor.mdoctor:main"
+21 -9
View File
@@ -7,19 +7,31 @@
- [x] Fix `mdoctor.py:19` - add required `directory` argument to `list_all_files()` call
- [x] Add argument parsing to both CLI tools
- [ ] **Fix bootstrap script issues**
- [ ] Remove duplicate function definitions (lines 239-286)
- [ ] Uncomment `install_requirements` call (line 349)
- [ ] Fix line 18: change `source "SCRIPT_DIR/src/utils.sh"` to `source "$SCRIPT_DIR/src/utils.sh"`
- [x] **Fix bootstrap script issues**
- [x] Removed duplicate function definitions (activate_environment, install_requirements)
- [x] Uncommented and enabled `install_requirements` call
- [x] Fixed line 31: `source "$SCRIPT_DIR/src/utils.sh"` with proper variable expansion
- [x] Improved `activate_environment()` to properly source activation scripts
- [x] Added macOS/homebrew support in `install_pyenv_dependencies()`
- [x] Added robust PATH handling for virtual environment activation
- [x] Added fallback activation paths
- [x] Improved error messages and validation
- [x] **Fix install_packages.sh**
- [x] Fixed missing `$` in `source "${SCRIPT_DIR}/utils.sh"`
- [x] Added `export` to PYENV_NAME variable assignment
- [x] Relaxed `check_pyenv()` to not require exact environment name match
- [x] Fixed missing `$` in `pushd "${SCRIPT_DIR}"` and suppressed output
- [x] Changed bootstrap to source install_packages.sh instead of running it in subshell
## 🟡 Structural Issues (High Priority)
- [ ] **Resolve package structure inconsistency**
- [ ] Decide: Keep flat structure OR move `librarian/` and `mdoctor/` into `fiona/` namespace
- [ ] Update all imports accordingly
- [ ] Update `pyproject.toml` entry points
- [x] **Resolve package structure inconsistency**
- [x] Decide: Keep flat structure (`fiona/`, `librarian/`, `mdoctor/` at same level)
- [x] Current structure confirmed: `src/fiona/` for entry point, `src/librarian/`, `src/mdoctor/` for modules
- [ ] **Consolidate requirements**
- [x] **Consolidate requirements**
- [x] Fix duplicate entry points in `pyproject.toml` (removed redundant `[project.entry-points.console_scripts]`)
- [ ] Delete `src/requirements.txt`, `src/librarian/requirements.txt`, `src/mdoctor/requirements.txt`
- [ ] Add all dependencies to `pyproject.toml`
- [ ] Add `dev-dependencies` section for pylint and other tools