fix: Enhance Alembic verification to test module execution #23
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD Pipeline | ||
| on: | ||
| push: | ||
| branches: [ main, develop, 'feature/**' ] | ||
| pull_request: | ||
| branches: [ main, develop ] | ||
| jobs: | ||
| backend-tests: | ||
| runs-on: ubuntu-latest | ||
| services: | ||
| postgres: | ||
| image: postgres:14 | ||
| env: | ||
| POSTGRES_PASSWORD: postgres | ||
| POSTGRES_DB: test_db | ||
| ports: | ||
| - 5432:5432 | ||
| options: >- | ||
| --health-cmd pg_isready | ||
| --health-interval 10s | ||
| --health-timeout 5s | ||
| --health-retries 5 | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Set up Python | ||
| uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: '3.12' | ||
| - name: Cache pip dependencies | ||
| uses: actions/cache@v3 | ||
| with: | ||
| path: ~/.cache/pip | ||
| key: ${{ runner.os }}-pip-${{ hashFiles('backend/requirements.txt') }} | ||
| restore-keys: | | ||
| ${{ runner.os }}-pip- | ||
| - name: Install dependencies | ||
| working-directory: ./backend | ||
| run: | | ||
| python -m pip install --upgrade pip setuptools wheel | ||
| # Install system dependencies | ||
| sudo apt-get update | ||
| sudo apt-get install -y tesseract-ocr | ||
| # Install core dependencies first with explicit paths | ||
| python -m pip install --upgrade 'sqlalchemy>=1.4.0,<2.0.0' | ||
| python -m pip install --upgrade --force-reinstall 'alembic>=1.12.0,<2.0.0' | ||
| python -m pip install --upgrade 'psycopg2-binary>=2.9.0,<3.0.0' | ||
| # Install requirements | ||
| python -m pip install -r requirements.txt | ||
| # Install test dependencies | ||
| python -m pip install pytest-cov flake8 httpx pytesseract pycountry PyYAML selenium pytest-mock loguru reportlab inputimeout | ||
| # Add local bin to PATH and verify installations | ||
| export PATH="$HOME/.local/bin:$PATH" | ||
| # Verify installations and paths | ||
| python -m pip list | ||
| # Verify Alembic installation and module accessibility | ||
| python << 'EOF' | ||
| import sys | ||
| import os | ||
| import subprocess | ||
| import importlib.util | ||
| print("Python version:", sys.version) | ||
| print("Python executable:", sys.executable) | ||
| print("PYTHONPATH:", os.getenv("PYTHONPATH")) | ||
| try: | ||
| import alembic | ||
| print("Alembic version:", alembic.__version__) | ||
| print("Alembic location:", alembic.__file__) | ||
| # Test if we can run alembic as a module | ||
| result = subprocess.run([sys.executable, '-m', 'alembic', '--version'], | ||
| capture_output=True, text=True) | ||
| print("Alembic module test output:", result.stdout) | ||
| if result.returncode != 0: | ||
| print("Alembic module test error:", result.stderr) | ||
| sys.exit(1) | ||
| print("Alembic module test successful") | ||
| except Exception as e: | ||
| print("Error:", str(e)) | ||
| sys.exit(1) | ||
| EOF | ||
| - name: Setup backend test environment | ||
| working-directory: ./backend | ||
| env: | ||
| POSTGRES_USER: postgres | ||
| POSTGRES_PASSWORD: postgres | ||
| POSTGRES_DB: test_db | ||
| run: | | ||
| echo "DEV_MODE=true" > .env | ||
| echo "DATABASE_URL=postgresql://postgres:postgres@localhost:5432/test_db" >> .env | ||
| echo "API_HOST=0.0.0.0" >> .env | ||
| echo "API_PORT=8000" >> .env | ||
| echo "[email protected]" >> .env | ||
| echo "INDEED_PASSWORD=testpass123" >> .env | ||
| # Initialize database schema | ||
| PGPASSWORD=postgres psql -h localhost -U postgres -d postgres -c 'DROP DATABASE IF EXISTS test_db;' | ||
| PGPASSWORD=postgres psql -h localhost -U postgres -d postgres -c 'CREATE DATABASE test_db;' | ||
| # Create initial schema | ||
| PGPASSWORD=postgres psql -h localhost -U postgres -d test_db << 'EOSQL' | ||
| CREATE TABLE IF NOT EXISTS users ( | ||
| id SERIAL PRIMARY KEY, | ||
| username VARCHAR(255) NOT NULL, | ||
| email VARCHAR(255) UNIQUE NOT NULL, | ||
| created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP | ||
| ); | ||
| CREATE TABLE IF NOT EXISTS verified_documents ( | ||
| id SERIAL PRIMARY KEY, | ||
| user_id INTEGER REFERENCES users(id), | ||
| document_type VARCHAR(50) NOT NULL, | ||
| document_number VARCHAR(255), | ||
| status VARCHAR(50) DEFAULT 'PENDING', | ||
| verification_date TIMESTAMP, | ||
| expiry_date TIMESTAMP, | ||
| created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP | ||
| ); | ||
| EOSQL | ||
| - name: Verify environment | ||
| working-directory: ./backend | ||
| env: | ||
| DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_db | ||
| PYTHONPATH: ${GITHUB_WORKSPACE}/backend | ||
| run: | | ||
| python -c "import sys; print('Python path:', sys.path)" | ||
| python -c "import alembic.config; print('Alembic configuration verified')" | ||
| python -c "import psycopg2; conn=psycopg2.connect('$DATABASE_URL'); print('Database connection verified')" | ||
| - name: Run migrations | ||
| working-directory: ./backend | ||
| env: | ||
| DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_db | ||
| PYTHONPATH: ${GITHUB_WORKSPACE}/backend | ||
| run: | | ||
| echo "Setting up Python environment..." | ||
| # Ensure backend directory is in Python path | ||
| export PYTHONPATH="${GITHUB_WORKSPACE}/backend:${PWD}:${PYTHONPATH}" | ||
| export PYTHONUNBUFFERED=1 | ||
| export PYTHONDONTWRITEBYTECODE=1 | ||
| echo "Current PYTHONPATH: $PYTHONPATH" | ||
| # Create Alembic directory structure | ||
| mkdir -p migrations/versions | ||
| # Initialize Alembic if not already initialized | ||
| if [ ! -f alembic.ini ]; then | ||
| python -m alembic init migrations | ||
| # Update alembic.ini with correct database URL | ||
| sed -i "s|sqlalchemy.url = .*|sqlalchemy.url = ${DATABASE_URL}|" alembic.ini | ||
| # Debug output | ||
| echo "Alembic initialization completed" | ||
| echo "Contents of alembic.ini:" | ||
| cat alembic.ini | ||
| fi | ||
| # Update env.py to use environment variable for database URL | ||
| cat > migrations/env.py << 'EOL' | ||
| from logging.config import fileConfig | ||
| from sqlalchemy import engine_from_config, pool, MetaData, Table, Column, Integer, String, ForeignKey, TIMESTAMP, text | ||
| from alembic import context | ||
| import os | ||
| config = context.config | ||
| if config.config_file_name is not None: | ||
| fileConfig(config.config_file_name) | ||
| def get_url(): | ||
| return os.getenv("DATABASE_URL") | ||
| # Create MetaData object | ||
| metadata = MetaData() | ||
| # Define tables | ||
| users = Table('users', metadata, | ||
| Column('id', Integer, primary_key=True), | ||
| Column('username', String(255), nullable=False), | ||
| Column('email', String(255), unique=True, nullable=False), | ||
| Column('created_at', TIMESTAMP, server_default=text('CURRENT_TIMESTAMP')) | ||
| ) | ||
| verified_documents = Table('verified_documents', metadata, | ||
| Column('id', Integer, primary_key=True), | ||
| Column('user_id', Integer, ForeignKey('users.id')), | ||
| Column('document_type', String(50), nullable=False), | ||
| Column('document_number', String(255)), | ||
| Column('status', String(50), server_default='PENDING'), | ||
| Column('verification_date', TIMESTAMP), | ||
| Column('expiry_date', TIMESTAMP), | ||
| Column('created_at', TIMESTAMP, server_default=text('CURRENT_TIMESTAMP')) | ||
| ) | ||
| target_metadata = metadata | ||
| def run_migrations_offline() -> None: | ||
| url = get_url() | ||
| context.configure( | ||
| url=url, | ||
| target_metadata=target_metadata, | ||
| literal_binds=True, | ||
| dialect_opts={"paramstyle": "named"}, | ||
| ) | ||
| with context.begin_transaction(): | ||
| context.run_migrations() | ||
| def run_migrations_online() -> None: | ||
| configuration = config.get_section(config.config_ini_section) | ||
| if configuration is None: | ||
| configuration = {} | ||
| configuration["sqlalchemy.url"] = get_url() | ||
| connectable = engine_from_config( | ||
| configuration, | ||
| prefix="sqlalchemy.", | ||
| poolclass=pool.NullPool, | ||
| ) | ||
| with connectable.connect() as connection: | ||
| context.configure( | ||
| connection=connection, | ||
| target_metadata=target_metadata | ||
| ) | ||
| with context.begin_transaction(): | ||
| context.run_migrations() | ||
| if context.is_offline_mode(): | ||
| run_migrations_offline() | ||
| else: | ||
| run_migrations_online() | ||
| EOL | ||
| # Debug output before migrations | ||
| echo "Verifying migrations setup..." | ||
| ls -la migrations/ | ||
| echo "Verifying migrations/versions directory..." | ||
| ls -la migrations/versions/ | ||
| echo "Current working directory: $(pwd)" | ||
| echo "Running initial migration..." | ||
| python -m alembic revision --autogenerate -m "Initial migration" | ||
| echo "Running migration upgrade..." | ||
| python -m alembic upgrade head | ||
| - name: Run tests with coverage | ||
| working-directory: ./backend | ||
| env: | ||
| DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_db | ||
| run: | | ||
| pytest tests/ -v --cov=src --cov-report=xml --junitxml=test-results/results.xml | ||
| - name: Run flake8 | ||
| working-directory: ./backend | ||
| run: | | ||
| flake8 src tests --max-line-length=100 | ||
| - name: Upload test results | ||
| uses: actions/upload-artifact@v4 | ||
| if: always() | ||
| with: | ||
| name: backend-test-results | ||
| path: | | ||
| backend/coverage.xml | ||
| backend/test-results/results.xml | ||
| frontend-tests: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - uses: pnpm/action-setup@v2 | ||
| with: | ||
| version: 8 | ||
| - name: Set up Node.js | ||
| uses: actions/setup-node@v3 | ||
| with: | ||
| node-version: '18' | ||
| cache: 'pnpm' | ||
| cache-dependency-path: './frontend/pnpm-lock.yaml' | ||
| - name: Install dependencies | ||
| working-directory: ./frontend | ||
| run: pnpm install | ||
| - name: Run tests | ||
| working-directory: ./frontend | ||
| run: CI=true pnpm test | ||
| - name: Upload test results | ||
| uses: actions/upload-artifact@v4 | ||
| if: always() | ||
| with: | ||
| name: frontend-test-results | ||
| path: frontend/test-results | ||
