fix: Simplify Alembic migration commands to use proper module execution #21
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD Pipeline | ||
| on: | ||
| push: | ||
| branches: [ main, develop, 'feature/**' ] | ||
| pull_request: | ||
| branches: [ main, develop ] | ||
| jobs: | ||
| backend-tests: | ||
| runs-on: ubuntu-latest | ||
| services: | ||
| postgres: | ||
| image: postgres:14 | ||
| env: | ||
| POSTGRES_PASSWORD: postgres | ||
| POSTGRES_DB: test_db | ||
| ports: | ||
| - 5432:5432 | ||
| options: >- | ||
| --health-cmd pg_isready | ||
| --health-interval 10s | ||
| --health-timeout 5s | ||
| --health-retries 5 | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Set up Python | ||
| uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: '3.12' | ||
| - name: Cache pip dependencies | ||
| uses: actions/cache@v3 | ||
| with: | ||
| path: ~/.cache/pip | ||
| key: ${{ runner.os }}-pip-${{ hashFiles('backend/requirements.txt') }} | ||
| restore-keys: | | ||
| ${{ runner.os }}-pip- | ||
| - name: Install dependencies | ||
| working-directory: ./backend | ||
| run: | | ||
| python -m pip install --upgrade pip setuptools wheel | ||
| # Install system dependencies | ||
| sudo apt-get update | ||
| sudo apt-get install -y tesseract-ocr | ||
| # Install core dependencies first with explicit paths | ||
| python -m pip install --upgrade 'sqlalchemy>=1.4.0,<2.0.0' | ||
| python -m pip install --upgrade --force-reinstall 'alembic>=1.12.0,<2.0.0' | ||
| python -m pip install --upgrade 'psycopg2-binary>=2.9.0,<3.0.0' | ||
| # Install requirements | ||
| python -m pip install -r requirements.txt | ||
| # Install test dependencies | ||
| python -m pip install pytest-cov flake8 httpx pytesseract pycountry PyYAML selenium pytest-mock loguru reportlab inputimeout | ||
| # Add local bin to PATH and verify installations | ||
| export PATH="$HOME/.local/bin:$PATH" | ||
| # Verify installations and paths | ||
| python -m pip list | ||
| # Verify Alembic installation and module accessibility | ||
| python << 'EOF' | ||
| import sys | ||
| import os | ||
| import importlib.util | ||
| print("Python version:", sys.version) | ||
| print("Python executable:", sys.executable) | ||
| print("PYTHONPATH:", os.getenv("PYTHONPATH")) | ||
| try: | ||
| import alembic | ||
| import alembic.__main__ | ||
| print("Alembic version:", alembic.__version__) | ||
| print("Alembic location:", alembic.__file__) | ||
| print("Alembic __main__ location:", alembic.__main__.__file__) | ||
| print("Alembic successfully imported") | ||
| except ImportError as e: | ||
| print("Error importing Alembic:", e) | ||
| exit(1) | ||
| EOF | ||
| - name: Setup backend test environment | ||
| working-directory: ./backend | ||
| env: | ||
| POSTGRES_USER: postgres | ||
| POSTGRES_PASSWORD: postgres | ||
| POSTGRES_DB: test_db | ||
| run: | | ||
| echo "DEV_MODE=true" > .env | ||
| echo "DATABASE_URL=postgresql://postgres:postgres@localhost:5432/test_db" >> .env | ||
| echo "API_HOST=0.0.0.0" >> .env | ||
| echo "API_PORT=8000" >> .env | ||
| echo "[email protected]" >> .env | ||
| echo "INDEED_PASSWORD=testpass123" >> .env | ||
| # Initialize database schema | ||
| PGPASSWORD=postgres psql -h localhost -U postgres -d postgres -c 'DROP DATABASE IF EXISTS test_db;' | ||
| PGPASSWORD=postgres psql -h localhost -U postgres -d postgres -c 'CREATE DATABASE test_db;' | ||
| # Create initial schema | ||
| PGPASSWORD=postgres psql -h localhost -U postgres -d test_db << 'EOSQL' | ||
| CREATE TABLE IF NOT EXISTS users ( | ||
| id SERIAL PRIMARY KEY, | ||
| username VARCHAR(255) NOT NULL, | ||
| email VARCHAR(255) UNIQUE NOT NULL, | ||
| created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP | ||
| ); | ||
| CREATE TABLE IF NOT EXISTS verified_documents ( | ||
| id SERIAL PRIMARY KEY, | ||
| user_id INTEGER REFERENCES users(id), | ||
| document_type VARCHAR(50) NOT NULL, | ||
| document_number VARCHAR(255), | ||
| status VARCHAR(50) DEFAULT 'PENDING', | ||
| verification_date TIMESTAMP, | ||
| expiry_date TIMESTAMP, | ||
| created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP | ||
| ); | ||
| EOSQL | ||
| - name: Verify environment | ||
| working-directory: ./backend | ||
| env: | ||
| DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_db | ||
| PYTHONPATH: ${GITHUB_WORKSPACE}/backend | ||
| run: | | ||
| python -c "import sys; print('Python path:', sys.path)" | ||
| python -c "import alembic.config; print('Alembic configuration verified')" | ||
| python -c "import psycopg2; conn=psycopg2.connect('$DATABASE_URL'); print('Database connection verified')" | ||
| - name: Run migrations | ||
| working-directory: ./backend | ||
| env: | ||
| DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_db | ||
| PYTHONPATH: ${GITHUB_WORKSPACE}/backend | ||
| run: | | ||
| echo "Setting up Python environment..." | ||
| # Ensure backend directory is in Python path | ||
| export PYTHONPATH="${GITHUB_WORKSPACE}/backend:${PWD}:${PYTHONPATH}" | ||
| echo "Current PYTHONPATH: $PYTHONPATH" | ||
| # Create Alembic directory structure | ||
| mkdir -p migrations/versions | ||
| # Initialize Alembic if not already initialized | ||
| if [ ! -f alembic.ini ]; then | ||
| python -m alembic init migrations | ||
| # Update alembic.ini with correct database URL | ||
| sed -i "s|sqlalchemy.url = .*|sqlalchemy.url = ${DATABASE_URL}|" alembic.ini | ||
| # Debug output | ||
| echo "Alembic initialization completed" | ||
| echo "Contents of alembic.ini:" | ||
| cat alembic.ini | ||
| fi | ||
| # Update env.py to use environment variable for database URL | ||
| cat > migrations/env.py << 'EOL' | ||
| from logging.config import fileConfig | ||
| from sqlalchemy import engine_from_config, pool, MetaData, Table, Column, Integer, String, ForeignKey, TIMESTAMP, text | ||
| from alembic import context | ||
| import os | ||
| config = context.config | ||
| if config.config_file_name is not None: | ||
| fileConfig(config.config_file_name) | ||
| def get_url(): | ||
| return os.getenv("DATABASE_URL") | ||
| # Create MetaData object | ||
| metadata = MetaData() | ||
| # Define tables | ||
| users = Table('users', metadata, | ||
| Column('id', Integer, primary_key=True), | ||
| Column('username', String(255), nullable=False), | ||
| Column('email', String(255), unique=True, nullable=False), | ||
| Column('created_at', TIMESTAMP, server_default=text('CURRENT_TIMESTAMP')) | ||
| ) | ||
| verified_documents = Table('verified_documents', metadata, | ||
| Column('id', Integer, primary_key=True), | ||
| Column('user_id', Integer, ForeignKey('users.id')), | ||
| Column('document_type', String(50), nullable=False), | ||
| Column('document_number', String(255)), | ||
| Column('status', String(50), server_default='PENDING'), | ||
| Column('verification_date', TIMESTAMP), | ||
| Column('expiry_date', TIMESTAMP), | ||
| Column('created_at', TIMESTAMP, server_default=text('CURRENT_TIMESTAMP')) | ||
| ) | ||
| target_metadata = metadata | ||
| def run_migrations_offline() -> None: | ||
| url = get_url() | ||
| context.configure( | ||
| url=url, | ||
| target_metadata=target_metadata, | ||
| literal_binds=True, | ||
| dialect_opts={"paramstyle": "named"}, | ||
| ) | ||
| with context.begin_transaction(): | ||
| context.run_migrations() | ||
| def run_migrations_online() -> None: | ||
| configuration = config.get_section(config.config_ini_section) | ||
| if configuration is None: | ||
| configuration = {} | ||
| configuration["sqlalchemy.url"] = get_url() | ||
| connectable = engine_from_config( | ||
| configuration, | ||
| prefix="sqlalchemy.", | ||
| poolclass=pool.NullPool, | ||
| ) | ||
| with connectable.connect() as connection: | ||
| context.configure( | ||
| connection=connection, | ||
| target_metadata=target_metadata | ||
| ) | ||
| with context.begin_transaction(): | ||
| context.run_migrations() | ||
| if context.is_offline_mode(): | ||
| run_migrations_offline() | ||
| else: | ||
| run_migrations_online() | ||
| EOL | ||
| # Debug output before migrations | ||
| echo "Verifying migrations setup..." | ||
| ls -la migrations/ | ||
| echo "Verifying migrations/versions directory..." | ||
| ls -la migrations/versions/ | ||
| echo "Current working directory: $(pwd)" | ||
| # Verify Python environment | ||
| echo "Verifying Python environment..." | ||
| python << 'EOF' | ||
| import sys | ||
| import os | ||
| import site | ||
| print("Python version:", sys.version) | ||
| print("Python executable:", sys.executable) | ||
| print("PYTHONPATH:", os.getenv("PYTHONPATH")) | ||
| print("Site packages:", site.getsitepackages()) | ||
| print("Working directory:", os.getcwd()) | ||
| print("Environment variables:") | ||
| for key in ["DATABASE_URL", "PYTHONPATH"]: | ||
| print(f" {key}:", os.getenv(key)) | ||
| EOF | ||
| # Verify database connection and environment | ||
| echo "Verifying database connection..." | ||
| python << 'EOF' | ||
| import os | ||
| import psycopg2 | ||
| try: | ||
| conn = psycopg2.connect(os.getenv("DATABASE_URL")) | ||
| print("Database connection successful") | ||
| conn.close() | ||
| except Exception as e: | ||
| print(f"Database connection error: {e}") | ||
| exit(1) | ||
| EOF | ||
| # Set up environment variables for both migration commands | ||
| export PYTHONPATH="${GITHUB_WORKSPACE}/backend:${PWD}:${PYTHONPATH}" | ||
| export PYTHONUNBUFFERED=1 | ||
| export PYTHONDONTWRITEBYTECODE=1 | ||
| echo "Running initial migration..." | ||
| python -m alembic revision --autogenerate -m "Initial migration" | ||
| echo "Running migration upgrade..." | ||
| python -m alembic upgrade head | ||
| - name: Run tests with coverage | ||
| working-directory: ./backend | ||
| env: | ||
| DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_db | ||
| run: | | ||
| pytest tests/ -v --cov=src --cov-report=xml --junitxml=test-results/results.xml | ||
| - name: Run flake8 | ||
| working-directory: ./backend | ||
| run: | | ||
| flake8 src tests --max-line-length=100 | ||
| - name: Upload test results | ||
| uses: actions/upload-artifact@v4 | ||
| if: always() | ||
| with: | ||
| name: backend-test-results | ||
| path: | | ||
| backend/coverage.xml | ||
| backend/test-results/results.xml | ||
| frontend-tests: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - uses: pnpm/action-setup@v2 | ||
| with: | ||
| version: 8 | ||
| - name: Set up Node.js | ||
| uses: actions/setup-node@v3 | ||
| with: | ||
| node-version: '18' | ||
| cache: 'pnpm' | ||
| cache-dependency-path: './frontend/pnpm-lock.yaml' | ||
| - name: Install dependencies | ||
| working-directory: ./frontend | ||
| run: pnpm install | ||
| - name: Run tests | ||
| working-directory: ./frontend | ||
| run: CI=true pnpm test | ||
| - name: Upload test results | ||
| uses: actions/upload-artifact@v4 | ||
| if: always() | ||
| with: | ||
| name: frontend-test-results | ||
| path: frontend/test-results | ||
