feat: enable remote pilot logging system #1228
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Basic Tests | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| defaults: | |
| run: | |
| shell: bash -el {0} | |
| jobs: | |
| shellcheck: | |
| runs-on: ubuntu-latest | |
| if: github.event_name != 'push' || github.repository == 'DIRACGrid/diracx' | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Run shellcheck | |
| # Excluded codes related to sourcing files | |
| # SC1090: Can't follow non-constant source | |
| # SC1091: Not following sourced file | |
| run: | | |
| find -name '*.sh' -print0 | xargs -0 -n1 shellcheck --exclude=SC1090,SC1091 --external-source | |
| pytest: | |
| name: Unit test - ${{ matrix.package }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - package: "./diracx-core" | |
| dependencies: "./diracx-testing" | |
| - package: "./diracx-db" | |
| dependencies: "./diracx-testing ./diracx-core" | |
| - package: "./diracx-routers" | |
| dependencies: "./diracx-testing ./diracx-core ./diracx-db" | |
| - package: "./diracx-client" | |
| dependencies: "./diracx-testing ./diracx-core" | |
| - package: "./diracx-api" | |
| dependencies: "./diracx-testing ./diracx-core ./diracx-client" | |
| - package: "./diracx-cli" | |
| dependencies: "./diracx-testing ./diracx-core ./diracx-client ./diracx-api" | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - uses: mamba-org/setup-micromamba@v1 | |
| with: | |
| # TODO: Use a conda environment file used for the diracx/base container image | |
| environment-name: test-env | |
| environment-file: environment.yml | |
| init-shell: bash | |
| post-cleanup: 'all' | |
| - name: Set up environment | |
| run: | | |
| pip install pytest-github-actions-annotate-failures | |
| pip install git+https://github.com/DIRACGrid/DIRAC.git@integration | |
| pip install ${{ matrix.dependencies }} | |
| - name: Run pytest | |
| run: | | |
| cd ${{ matrix.package }} | |
| pip install .[testing] | |
| pytest --cov-report=xml:coverage.xml --junitxml=report.xml | |
| - name: Run mypy | |
| run: | | |
| pip install mypy ${{ matrix.package }}[types] | |
| mypy ${{ matrix.package }}/src | |
| - name: Upload coverage report | |
| uses: codecov/[email protected] | |
| pytest-integration: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - uses: mamba-org/setup-micromamba@v1 | |
| with: | |
| environment-file: environment.yml | |
| init-shell: bash | |
| post-cleanup: 'all' | |
| - name: Set up environment | |
| run: | | |
| pip install pytest-github-actions-annotate-failures | |
| pip install git+https://github.com/DIRACGrid/DIRAC.git@integration | |
| pip install ./diracx-core/[testing] ./diracx-api/[testing] ./diracx-cli/[testing] ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-db/[testing] ./diracx-testing/ | |
| - name: Start demo | |
| run: | | |
| git clone https://github.com/DIRACGrid/diracx-charts.git ../diracx-charts | |
| ../diracx-charts/run_demo.sh --enable-open-telemetry --enable-coverage --exit-when-done --set-value developer.autoReload=false --ci-values ../diracx-charts/demo/ci_values.yaml $PWD | |
| - name: Debugging information | |
| run: | | |
| DIRACX_DEMO_DIR=$PWD/../diracx-charts/.demo | |
| export KUBECONFIG=${DIRACX_DEMO_DIR}/kube.conf | |
| export PATH=${DIRACX_DEMO_DIR}:$PATH | |
| kubectl get pods | |
| for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|rabbitmq|opensearch)'); do | |
| echo "${pod_name}" | |
| kubectl describe pod/"${pod_name}" || true | |
| for container_name in $(kubectl get pods $pod_name -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}'); do | |
| echo $pod_name $container_name | |
| kubectl logs "${pod_name}" -c "${container_name}" || true | |
| done | |
| done | |
| if [ ! -f "${DIRACX_DEMO_DIR}/.success" ]; then | |
| cat "${DIRACX_DEMO_DIR}/.failed" | |
| exit 1 | |
| fi | |
| - name: Run pytest | |
| run: | | |
| pytest --demo-dir=../diracx-charts/ --cov-report=xml:coverage-pytest.xml --junitxml=report.xml | |
| - name: Collect demo coverage | |
| run: | | |
| DIRACX_DEMO_DIR=$PWD/../diracx-charts/.demo | |
| export KUBECONFIG=${DIRACX_DEMO_DIR}/kube.conf | |
| export PATH=${DIRACX_DEMO_DIR}:$PATH | |
| # Shutdown the pods so we collect coverage data | |
| for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|rabbitmq|opensearch)'); do | |
| kubectl delete pod/"${pod_name}" | |
| done | |
| # Combine the coverage data from the demo and make an XML report | |
| coverage_data=$(mktemp) | |
| sudo chown -R $(id -u) "${DIRACX_DEMO_DIR}"/coverage-reports/ | |
| coverage combine --keep --data-file "${coverage_data}" "${DIRACX_DEMO_DIR}"/coverage-reports/* | |
| # coverage can't handle having multiple src directories, so we need to make a fake one with symlinks | |
| fake_module=$(mktemp -d) | |
| mkdir -p "${fake_module}/src/diracx" | |
| for fn in "${PWD}"/*/src/diracx/*; do | |
| ln -sf "${fn}" "${fake_module}/src/diracx/$(basename "${fn}")" | |
| done | |
| sed -i "s@source =@source =\n ${fake_module}/src@g" .coveragerc | |
| cat .coveragerc | |
| coverage xml -o coverage-demo.xml --data-file "${coverage_data}" | |
| - name: Upload coverage report | |
| uses: codecov/[email protected] | |
| with: | |
| files: ./coverage-pytest.xml,./coverage-demo.xml | |
| client-generation: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - uses: mamba-org/setup-micromamba@v1 | |
| with: | |
| environment-file: environment.yml | |
| init-shell: bash | |
| post-cleanup: 'all' | |
| - name: Set up environment | |
| run: | | |
| micromamba install -c conda-forge nodejs pre-commit | |
| pip install git+https://github.com/DIRACGrid/DIRAC.git@integration | |
| pip install ./diracx-core/ ./diracx-api/ ./diracx-cli/ -e ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-db/ ./diracx-testing/ | |
| npm install -g autorest | |
| - name: Run autorest | |
| run: | | |
| autorest --python --help | |
| $HOME/.autorest/\@autorest_python\@*/node_modules/\@autorest/python/venv/bin/python -m pip install --upgrade setuptools | |
| pytest --no-cov --regenerate-client diracx-client/tests/test_regenerate.py |