main.yml

name: Basic Tests

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main


defaults:
  run:
    shell: bash -el {0}

jobs:

  shellcheck:
    runs-on: ubuntu-latest
    if: github.event_name != 'push' || github.repository == 'DIRACGrid/diracx'
    timeout-minutes: 30

    steps:
    - uses: actions/checkout@v4
    - name: Run shellcheck
      # Excluded codes related to sourcing files
      #     SC1090: Can't follow non-constant source
      #     SC1091: Not following sourced file
      run: |
        find -name '*.sh' -print0 | xargs -0 -n1 shellcheck --exclude=SC1090,SC1091 --external-source

  pytest:
    name: Unit test - ${{ matrix.package }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - package: "./diracx-core"
            dependencies: "./diracx-testing"
          - package: "./diracx-db"
            dependencies: "./diracx-testing ./diracx-core"
          - package: "./diracx-routers"
            dependencies: "./diracx-testing ./diracx-core ./diracx-db"
          - package: "./diracx-client"
            dependencies: "./diracx-testing ./diracx-core"
          - package: "./diracx-api"
            dependencies: "./diracx-testing ./diracx-core ./diracx-client"
          - package: "./diracx-cli"
            dependencies: "./diracx-testing ./diracx-core ./diracx-client ./diracx-api"
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - uses: mamba-org/setup-micromamba@v1
        with:
          # TODO: Use a conda environment file used for the diracx/base container image
          environment-name: test-env
          environment-file: environment.yml
          init-shell: bash
          post-cleanup: 'all'
      - name: Set up environment
        run: |
          pip install pytest-github-actions-annotate-failures
          pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
          pip install ${{ matrix.dependencies }}
      - name: Run pytest
        run: |
          cd ${{ matrix.package }}
          pip install .[testing]
          pytest --cov-report=xml:coverage.xml --junitxml=report.xml
      - name: Run mypy
        run: |
          pip install mypy ${{ matrix.package }}[types]
          mypy ${{ matrix.package }}/src
      - name: Upload coverage report
        uses: codecov/codecov-action@v4.5.0

  pytest-integration:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: environment.yml
          init-shell: bash
          post-cleanup: 'all'
      - name: Set up environment
        run: |
          pip install pytest-github-actions-annotate-failures
          pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
          pip install ./diracx-core/[testing] ./diracx-api/[testing] ./diracx-cli/[testing] ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-db/[testing] ./diracx-testing/
      - name: Start demo
        run: |
          git clone https://github.com/DIRACGrid/diracx-charts.git ../diracx-charts
          ../diracx-charts/run_demo.sh --enable-open-telemetry --enable-coverage --exit-when-done --set-value developer.autoReload=false --ci-values ../diracx-charts/demo/ci_values.yaml $PWD
      - name: Debugging information
        run: |
          DIRACX_DEMO_DIR=$PWD/../diracx-charts/.demo
          export KUBECONFIG=${DIRACX_DEMO_DIR}/kube.conf
          export PATH=${DIRACX_DEMO_DIR}:$PATH
          kubectl get pods
          for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|rabbitmq|opensearch)'); do
            echo "${pod_name}"
            kubectl describe pod/"${pod_name}" || true
            for container_name in $(kubectl get pods $pod_name -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}'); do
              echo $pod_name $container_name
              kubectl logs "${pod_name}" -c "${container_name}" || true
            done
          done
          if [ ! -f "${DIRACX_DEMO_DIR}/.success" ]; then
            cat "${DIRACX_DEMO_DIR}/.failed"
            exit 1
          fi
      - name: Run pytest
        run: |
          pytest --demo-dir=../diracx-charts/ --cov-report=xml:coverage-pytest.xml --junitxml=report.xml
      - name: Collect demo coverage
        run: |
          DIRACX_DEMO_DIR=$PWD/../diracx-charts/.demo
          export KUBECONFIG=${DIRACX_DEMO_DIR}/kube.conf
          export PATH=${DIRACX_DEMO_DIR}:$PATH
          # Shutdown the pods so we collect coverage data
          for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|rabbitmq|opensearch)'); do
            kubectl delete pod/"${pod_name}"
          done
          # Combine the coverage data from the demo and make an XML report
          coverage_data=$(mktemp)
          sudo chown -R $(id -u) "${DIRACX_DEMO_DIR}"/coverage-reports/
          coverage combine --keep --data-file "${coverage_data}" "${DIRACX_DEMO_DIR}"/coverage-reports/*

          # coverage can't handle having multiple src directories, so we need to make a fake one with symlinks
          fake_module=$(mktemp -d)
          mkdir -p "${fake_module}/src/diracx"
          for fn in "${PWD}"/*/src/diracx/*; do
            ln -sf "${fn}" "${fake_module}/src/diracx/$(basename "${fn}")"
          done
          sed -i "s@source =@source =\n    ${fake_module}/src@g" .coveragerc
          cat .coveragerc

          coverage xml -o coverage-demo.xml --data-file "${coverage_data}"
      - name: Upload coverage report
        uses: codecov/codecov-action@v4.5.0
        with:
          files: ./coverage-pytest.xml,./coverage-demo.xml

  client-generation:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: environment.yml
          init-shell: bash
          post-cleanup: 'all'
      - name: Set up environment
        run: |
          micromamba install -c conda-forge nodejs pre-commit
          pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
          pip install ./diracx-core/ ./diracx-api/ ./diracx-cli/ -e ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-db/ ./diracx-testing/
          npm install -g autorest
      - name: Run autorest
        run: |
          autorest --python --help
          $HOME/.autorest/\@autorest_python\@*/node_modules/\@autorest/python/venv/bin/python -m pip install --upgrade setuptools
          pytest --no-cov --regenerate-client diracx-client/tests/test_regenerate.py