Skip to content
CI

Merge pull request #134 from ral-facilities/DSEGOG-337-in-progress-dir #734

Sign in to view logs

Merge pull request #134 from ral-facilities/DSEGOG-337-in-progress-dir

Merge pull request #134 from ral-facilities/DSEGOG-337-in-progress-dir #734

Workflow file for this run

.github/workflows/ci.yml at d631d9d
name: CI
on:
workflow_dispatch:
pull_request:
push:
branches:
- main
jobs:
tests:
strategy:
fail-fast: false
matrix:
os: [ubuntu-20.04, ubuntu-22.04]
python-version: ["3.8", "3.9", "3.10", "3.11"]
exclude:
- os: ubuntu-22.04
python-version: "3.8"
- os: ubuntu-22.04
python-version: "3.9"
- os: ubuntu-20.04
python-version: "3.10"
- os: ubuntu-20.04
python-version: "3.11"
runs-on: ${{ matrix.os }}
name: Python ${{ matrix.python-version }} Tests
steps:
- name: Checkout OperationsGateway API
uses: actions/checkout@v3
# Install dependencies of python-ldap
- name: Install python-ldap dependencies
run: |
sudo apt-get update
sudo apt-get install -y libsasl2-dev python${{ matrix.python-version }}-dev libldap2-dev libssl-dev
# Setup Python and environment dependencies (via cache)
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Load Pip cache
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('.github/ci_requirements.txt') }}
- name: Install Poetry & Nox
run: pip install -r .github/ci_requirements.txt
# Install and start MongoDB
- name: Start MongoDB
uses: supercharge/[email protected]
with:
mongodb-version: '7.0'
# Used to install mongoimport when Ubuntu 22.04 is used, identified at https://github.com/actions/runner-images/issues/6626#issuecomment-1327744126
- name: Install MongoDB Database Tools
if: matrix.os == 'ubuntu-22.04'
run: |
sudo apt-get update
sudo apt-get install -y wget gnupg
wget -qO - https://www.mongodb.org/static/pgp/server-7.0.asc | sudo apt-key add -
echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-7.0.list
sudo apt-get update
sudo apt-get install -y mongodb-database-tools
# Read the database name from the config file and store it in an environment variable
- name: Get database name from ci_config.yml
run: echo "DATABASE_NAME=$(grep database_name .github/ci_config.yml | cut -d ':' -f 2 | tr -d '[:space:]')" >> $GITHUB_ENV
# Load Poetry virtual environment dependencies and install API dependencies
- name: Load Poetry cache
uses: actions/cache@v3
with:
path: ~/.cache/pypoetry/virtualenvs
key: ${{ runner.os }}-poetry-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
- name: Install dependencies
run: poetry install --without simulated-data
# Setup minio and create bucket
- name: Setup minio
run: |
docker run -d -p 9000:9000 --name minio \
-e "MINIO_ACCESS_KEY=minioadmin" \
-e "MINIO_SECRET_KEY=minioadmin" \
-v /tmp/data:/data \
-v /tmp/config:/root/.minio \
minio/minio server /data
export AWS_ACCESS_KEY_ID=minioadmin
export AWS_SECRET_ACCESS_KEY=minioadmin
export AWS_EC2_METADATA_DISABLED=true
aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://og-actions-test
- name: Move CI config.yml to correct place
run: cp .github/ci_config.yml operationsgateway_api/config.yml
- name: Setup logging configuration
run: cp operationsgateway_api/logging.ini.example operationsgateway_api/logging.ini
- name: Create log file
run: touch "$GITHUB_WORKSPACE/logs.log"
- name: Create SSH private key file for auth
run: 'echo "$SSH_KEY_PRIVATE" > /home/runner/work/operationsgateway-api/id_rsa'
shell: bash
env:
SSH_KEY_PRIVATE: ${{secrets.SSH_PRIVATE_KEY_FOR_AUTH_OPENSSH}}
- name: Set permissions on private key file
run: chmod 600 /home/runner/work/operationsgateway-api/id_rsa
- name: Create SSH public key file for auth
run: 'echo "$SSH_KEY_PUBLIC" > /home/runner/work/operationsgateway-api/id_rsa.pub'
shell: bash
env:
SSH_KEY_PUBLIC: ${{secrets.SSH_PUBLIC_KEY_FOR_AUTH_OPENSSH}}
# Setup steps for Echo ingestion script
- name: Configure echo access key
run: yq -i ".echo.access_key = \"$ECHO_S3_ACCESS_KEY\"" .github/ci_ingest_echo_config.yml
env:
ECHO_S3_ACCESS_KEY: ${{secrets.ECHO_S3_ACCESS_KEY}}
- name: Configure echo secret key
run: yq -i ".echo.secret_key = \"$ECHO_S3_SECRET_KEY\"" .github/ci_ingest_echo_config.yml
env:
ECHO_S3_SECRET_KEY: ${{secrets.ECHO_S3_SECRET_KEY}}
- name: Copy config for Echo Ingest script to correct place
run: cp .github/ci_ingest_echo_config.yml util/realistic_data/config.yml
- name: Run Echo Ingest script
run: poetry run python util/realistic_data/ingest_echo_data.py
- name: Load Poetry cache for Nox tests session
uses: actions/cache@v3
with:
path: /home/runner/work/operationsgateway-api/operationsgateway-api/.nox/tests*
key: ${{ runner.os }}-poetry-nox-tests-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
# Run Nox tests session, saves and uploads a coverage report to codecov
- name: Run Nox tests session
run: nox -p ${{ matrix.python-version }} -s tests -- --cov=operationsgateway_api --cov-report=xml
- name: Upload code coverage report
if: matrix.python-version == '3.8'
uses: codecov/codecov-action@v3
with:
token: ${{ secrets.CODECOV_TOKEN }}
linting:
runs-on: ubuntu-latest
name: Code Linting
steps:
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: "3.8.14"
- name: Checkout OperationsGateway API
uses: actions/checkout@v3
- name: Load Pip cache
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('.github/ci_requirements.txt') }}
- name: Install Poetry & Nox
run: pip install -r .github/ci_requirements.txt
- name: Load Poetry cache for Nox lint session
uses: actions/cache@v3
with:
path: /home/runner/work/operationsgateway-api/operationsgateway-api/.nox/lint*
key: ${{ runner.os }}-poetry-nox-lint-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
- name: Run Nox lint session
run: nox -s lint
formatting:
runs-on: ubuntu-latest
name: Code Formatting
steps:
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: "3.8.14"
- name: Checkout OperationsGateway API
uses: actions/checkout@v3
- name: Load Pip cache
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('.github/ci_requirements.txt') }}
- name: Install Poetry & Nox
run: pip install -r .github/ci_requirements.txt
- name: Load Poetry cache for Nox black session
uses: actions/cache@v3
with:
path: /home/runner/work/operationsgateway-api/operationsgateway-api/.nox/black*
key: ${{ runner.os }}-poetry-nox-black-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
- name: Run Nox black session
run: nox -s black
safety:
runs-on: ubuntu-latest
name: Dependency Safety
steps:
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: "3.8.14"
- name: Checkout OperationsGateway API
uses: actions/checkout@v3
- name: Load Pip cache
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('.github/ci_requirements.txt') }}
- name: Install Poetry & Nox
run: pip install -r .github/ci_requirements.txt
- name: Load Poetry cache for Nox safety session
uses: actions/cache@v3
with:
path: /home/runner/work/operationsgateway-api/operationsgateway-api/.nox/safety*
key: ${{ runner.os }}-poetry-nox-safety-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
- name: Run Nox safety session
run: nox -s safety