ci.yml

name: CI
on:
  workflow_dispatch:
  pull_request:
  push:
    branches:
      - main

jobs:
  tests:
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-20.04, ubuntu-22.04]
        python-version: ["3.8", "3.9", "3.10", "3.11"]
        exclude:
          - os: ubuntu-22.04
            python-version: "3.8"
          - os: ubuntu-22.04
            python-version: "3.9"
          - os: ubuntu-20.04
            python-version: "3.10"
          - os: ubuntu-20.04
            python-version: "3.11"
    runs-on: ${{ matrix.os }}

    name: Python ${{ matrix.python-version }} Tests
    steps:
      - name: Checkout OperationsGateway API
        uses: actions/checkout@v3

      # Install dependencies of python-ldap
      - name: Install python-ldap dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y libsasl2-dev python${{ matrix.python-version }}-dev libldap2-dev libssl-dev

      # Setup Python and environment dependencies (via cache)
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Load Pip cache
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('.github/ci_requirements.txt') }}
      - name: Install Poetry & Nox
        run: pip install -r .github/ci_requirements.txt

      # Install and start MongoDB
      - name: Start MongoDB
        uses: supercharge/mongodb-github-action@1.10.0
        with:
          mongodb-version: '7.0'

      # Used to install mongoimport when Ubuntu 22.04 is used, identified at https://github.com/actions/runner-images/issues/6626#issuecomment-1327744126
      - name: Install MongoDB Database Tools
        if: matrix.os == 'ubuntu-22.04'
        run: |
          sudo apt-get update
          sudo apt-get install -y wget gnupg
          wget -qO - https://www.mongodb.org/static/pgp/server-7.0.asc | sudo apt-key add -
          echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-7.0.list
          sudo apt-get update
          sudo apt-get install -y mongodb-database-tools

      # Read the database name from the config file and store it in an environment variable
      - name: Get database name from ci_config.yml
        run: echo "DATABASE_NAME=$(grep database_name .github/ci_config.yml | cut -d ':' -f 2 | tr -d '[:space:]')" >> $GITHUB_ENV

      # Load Poetry virtual environment dependencies and install API dependencies
      - name: Load Poetry cache
        uses: actions/cache@v3
        with:
          path: ~/.cache/pypoetry/virtualenvs
          key: ${{ runner.os }}-poetry-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
      - name: Install dependencies
        run: poetry install --without simulated-data

      # Setup minio and create bucket
      - name: Setup minio
        run: |
          docker run -d -p 9000:9000 --name minio \
                     -e "MINIO_ACCESS_KEY=minioadmin" \
                     -e "MINIO_SECRET_KEY=minioadmin" \
                     -v /tmp/data:/data \
                     -v /tmp/config:/root/.minio \
                     minio/minio server /data
          export AWS_ACCESS_KEY_ID=minioadmin
          export AWS_SECRET_ACCESS_KEY=minioadmin
          export AWS_EC2_METADATA_DISABLED=true
          aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://og-actions-test

      - name: Move CI config.yml to correct place
        run: cp .github/ci_config.yml operationsgateway_api/config.yml
      - name: Setup logging configuration
        run: cp operationsgateway_api/logging.ini.example operationsgateway_api/logging.ini
      - name: Create log file
        run: touch "$GITHUB_WORKSPACE/logs.log"

      - name: Create SSH private key file for auth
        run: 'echo "$SSH_KEY_PRIVATE" > /home/runner/work/operationsgateway-api/id_rsa'
        shell: bash
        env:
          SSH_KEY_PRIVATE: ${{secrets.SSH_PRIVATE_KEY_FOR_AUTH_OPENSSH}}

      - name: Set permissions on private key file
        run: chmod 600 /home/runner/work/operationsgateway-api/id_rsa

      - name: Create SSH public key file for auth
        run: 'echo "$SSH_KEY_PUBLIC" > /home/runner/work/operationsgateway-api/id_rsa.pub'
        shell: bash
        env:
          SSH_KEY_PUBLIC: ${{secrets.SSH_PUBLIC_KEY_FOR_AUTH_OPENSSH}}

      # Setup steps for Echo ingestion script
      - name: Configure echo access key
        run: yq -i ".echo.access_key = \"$ECHO_S3_ACCESS_KEY\"" .github/ci_ingest_echo_config.yml
        env:
          ECHO_S3_ACCESS_KEY: ${{secrets.ECHO_S3_ACCESS_KEY}}
      - name: Configure echo secret key
        run: yq -i ".echo.secret_key = \"$ECHO_S3_SECRET_KEY\"" .github/ci_ingest_echo_config.yml
        env:
          ECHO_S3_SECRET_KEY: ${{secrets.ECHO_S3_SECRET_KEY}}
      - name: Copy config for Echo Ingest script to correct place
        run: cp .github/ci_ingest_echo_config.yml util/realistic_data/config.yml

      - name: Run Echo Ingest script
        run: poetry run python util/realistic_data/ingest_echo_data.py

      - name: Load Poetry cache for Nox tests session
        uses: actions/cache@v3
        with:
          path: /home/runner/work/operationsgateway-api/operationsgateway-api/.nox/tests*
          key: ${{ runner.os }}-poetry-nox-tests-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}

      # Run Nox tests session, saves and uploads a coverage report to codecov
      - name: Run Nox tests session
        run: nox -p ${{ matrix.python-version }} -s tests -- --cov=operationsgateway_api --cov-report=xml
      - name: Upload code coverage report
        if: matrix.python-version == '3.8'
        uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }}


  linting:
    runs-on: ubuntu-latest
    name: Code Linting
    steps:
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8.14"
      - name: Checkout OperationsGateway API
        uses: actions/checkout@v3

      - name: Load Pip cache
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('.github/ci_requirements.txt') }}
      - name: Install Poetry & Nox
        run: pip install -r .github/ci_requirements.txt

      - name: Load Poetry cache for Nox lint session
        uses: actions/cache@v3
        with:
          path: /home/runner/work/operationsgateway-api/operationsgateway-api/.nox/lint*
          key: ${{ runner.os }}-poetry-nox-lint-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}

      - name: Run Nox lint session
        run: nox -s lint


  formatting:
    runs-on: ubuntu-latest
    name: Code Formatting
    steps:
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8.14"
      - name: Checkout OperationsGateway API
        uses: actions/checkout@v3

      - name: Load Pip cache
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('.github/ci_requirements.txt') }}
      - name: Install Poetry & Nox
        run: pip install -r .github/ci_requirements.txt

      - name: Load Poetry cache for Nox black session
        uses: actions/cache@v3
        with:
          path: /home/runner/work/operationsgateway-api/operationsgateway-api/.nox/black*
          key: ${{ runner.os }}-poetry-nox-black-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}

      - name: Run Nox black session
        run: nox -s black


  safety:
    runs-on: ubuntu-latest
    name: Dependency Safety
    steps:
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8.14"
      - name: Checkout OperationsGateway API
        uses: actions/checkout@v3

      - name: Load Pip cache
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('.github/ci_requirements.txt') }}
      - name: Install Poetry & Nox
        run: pip install -r .github/ci_requirements.txt

      - name: Load Poetry cache for Nox safety session
        uses: actions/cache@v3
        with:
          path: /home/runner/work/operationsgateway-api/operationsgateway-api/.nox/safety*
          key: ${{ runner.os }}-poetry-nox-safety-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}

      - name: Run Nox safety session
        run: nox -s safety