Migrate CI to protected runners and JFrog PyPI proxy #239
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Code Coverage | |
| permissions: | |
| contents: read | |
| id-token: write | |
| on: [pull_request, workflow_dispatch] | |
| jobs: | |
| test-with-coverage: | |
| runs-on: | |
| group: databricks-protected-runner-group | |
| labels: linux-ubuntu-latest | |
| environment: azure-prod | |
| env: | |
| DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }} | |
| DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }} | |
| DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} | |
| DATABRICKS_CATALOG: peco | |
| DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }} | |
| steps: | |
| #---------------------------------------------- | |
| # check-out repo and set-up python | |
| #---------------------------------------------- | |
| - name: Check out repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up python | |
| id: setup-python | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: "3.10" | |
| #---------------------------------------------- | |
| # ----- configure JFrog PyPI proxy ----- | |
| #---------------------------------------------- | |
| - name: Setup JFrog | |
| uses: ./.github/actions/setup-jfrog | |
| #---------------------------------------------- | |
| # ----- install system dependencies ----- | |
| #---------------------------------------------- | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libkrb5-dev | |
| #---------------------------------------------- | |
| # ----- install & configure poetry ----- | |
| #---------------------------------------------- | |
| - name: Install Poetry | |
| run: | | |
| pip install poetry==2.2.1 | |
| poetry config virtualenvs.create true | |
| poetry config virtualenvs.in-project true | |
| poetry config installer.parallel true | |
| - name: Configure Poetry for JFrog | |
| run: | | |
| poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple | |
| poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" | |
| poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple | |
| poetry lock --no-update | |
| #---------------------------------------------- | |
| # load cached venv if cache exists | |
| #---------------------------------------------- | |
| - name: Load cached venv | |
| id: cached-poetry-dependencies | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: .venv | |
| key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} | |
| #---------------------------------------------- | |
| # install dependencies if cache does not exist | |
| #---------------------------------------------- | |
| - name: Install dependencies | |
| if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' | |
| run: poetry install --no-interaction --no-root | |
| #---------------------------------------------- | |
| # install your root project, if required | |
| #---------------------------------------------- | |
| - name: Install Kerberos system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libkrb5-dev | |
| - name: Install library | |
| run: poetry install --no-interaction --all-extras | |
| #---------------------------------------------- | |
| # run parallel tests with coverage | |
| #---------------------------------------------- | |
| - name: Run parallel tests with coverage | |
| continue-on-error: false | |
| run: | | |
| poetry run pytest tests/unit tests/e2e \ | |
| -m "not serial" \ | |
| -n auto \ | |
| --cov=src \ | |
| --cov-report=xml \ | |
| --cov-report=term \ | |
| -v | |
| #---------------------------------------------- | |
| # run telemetry tests with coverage (isolated) | |
| #---------------------------------------------- | |
| - name: Run telemetry tests with coverage (isolated) | |
| continue-on-error: false | |
| run: | | |
| # Run test_concurrent_telemetry.py separately for isolation | |
| poetry run pytest tests/e2e/test_concurrent_telemetry.py \ | |
| --cov=src \ | |
| --cov-append \ | |
| --cov-report=xml \ | |
| --cov-report=term \ | |
| -v | |
| #---------------------------------------------- | |
| # check for coverage override | |
| #---------------------------------------------- | |
| - name: Check for coverage override | |
| id: override | |
| env: | |
| PR_BODY: ${{ github.event.pull_request.body }} | |
| run: | | |
| OVERRIDE_COMMENT=$(echo "$PR_BODY" | grep -E "SKIP_COVERAGE_CHECK\s*=" || echo "") | |
| if [ -n "$OVERRIDE_COMMENT" ]; then | |
| echo "override=true" >> $GITHUB_OUTPUT | |
| REASON=$(echo "$OVERRIDE_COMMENT" | sed -E 's/.*SKIP_COVERAGE_CHECK\s*=\s*(.+)/\1/') | |
| echo "reason=$REASON" >> $GITHUB_OUTPUT | |
| echo "Coverage override found in PR description: $REASON" | |
| else | |
| echo "override=false" >> $GITHUB_OUTPUT | |
| echo "No coverage override found" | |
| fi | |
| #---------------------------------------------- | |
| # check coverage percentage | |
| #---------------------------------------------- | |
| - name: Check coverage percentage | |
| if: steps.override.outputs.override == 'false' | |
| run: | | |
| COVERAGE_FILE="coverage.xml" | |
| if [ ! -f "$COVERAGE_FILE" ]; then | |
| echo "ERROR: Coverage file not found at $COVERAGE_FILE" | |
| exit 1 | |
| fi | |
| # Install xmllint if not available | |
| if ! command -v xmllint &> /dev/null; then | |
| sudo apt-get update && sudo apt-get install -y libxml2-utils | |
| fi | |
| COVERED=$(xmllint --xpath "string(//coverage/@lines-covered)" "$COVERAGE_FILE") | |
| TOTAL=$(xmllint --xpath "string(//coverage/@lines-valid)" "$COVERAGE_FILE") | |
| PERCENTAGE=$(python3 -c "covered=${COVERED}; total=${TOTAL}; print(round((covered/total)*100, 2))") | |
| echo "Branch Coverage: $PERCENTAGE%" | |
| echo "Required Coverage: 85%" | |
| # Use Python to compare the coverage with 85 | |
| python3 -c "import sys; sys.exit(0 if float('$PERCENTAGE') >= 85 else 1)" | |
| if [ $? -eq 1 ]; then | |
| echo "ERROR: Coverage is $PERCENTAGE%, which is less than the required 85%" | |
| exit 1 | |
| else | |
| echo "SUCCESS: Coverage is $PERCENTAGE%, which meets the required 85%" | |
| fi | |
| #---------------------------------------------- | |
| # coverage enforcement summary | |
| #---------------------------------------------- | |
| - name: Coverage enforcement summary | |
| env: | |
| OVERRIDE: ${{ steps.override.outputs.override }} | |
| REASON: ${{ steps.override.outputs.reason }} | |
| run: | | |
| if [ "$OVERRIDE" == "true" ]; then | |
| echo "⚠️ Coverage checks bypassed: $REASON" | |
| echo "Please ensure this override is justified and temporary" | |
| else | |
| echo "✅ Coverage checks enforced - minimum 85% required" | |
| fi |