From dea430649dd8c940e4db973577f9c8e2493b40b2 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Wed, 10 Apr 2024 14:32:30 -0700 Subject: [PATCH] chore: [proposal] de-matrix python-version in GHAs (#27906) --- .asf.yaml | 19 ++-- .github/actions/setup-backend/action.yml | 37 +++++-- .github/workflows/docker.yml | 2 + .github/workflows/no-op.yml | 96 +++++++++++++++++++ .github/workflows/pre-commit.yml | 3 - .github/workflows/superset-cli.yml | 3 - .github/workflows/superset-helm-lint.yml | 5 +- .../superset-python-integrationtest.yml | 15 +-- .github/workflows/superset-python-misc.yml | 10 -- .../workflows/superset-python-presto-hive.yml | 6 -- .../workflows/superset-python-unittest.yml | 2 +- .github/workflows/superset-translations.yml | 5 - 12 files changed, 141 insertions(+), 62 deletions(-) create mode 100644 .github/workflows/no-op.yml diff --git a/.asf.yaml b/.asf.yaml index 9b2bfae806ad..14f02fb18ec4 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -67,15 +67,16 @@ github: - cypress-matrix (2, chrome) - cypress-matrix (3, chrome) - frontend-build - - pre-commit (3.10) - - python-lint (3.10) - - test-mysql (3.10) - - test-postgres (3.10) - - test-postgres (3.11) - - test-postgres-hive (3.10) - - test-postgres-presto (3.10) - - test-sqlite (3.10) - - unit-tests (3.10) + - pre-commit + - python-lint + - test-mysql + - test-postgres (current) + - test-postgres (next) + - test-postgres-hive + - test-postgres-presto + - test-sqlite + - unit-tests (current) + - unit-tests (next) required_pull_request_reviews: dismiss_stale_reviews: false diff --git a/.github/actions/setup-backend/action.yml b/.github/actions/setup-backend/action.yml index 5f6ba8d1a02f..4f97916dbee7 100644 --- a/.github/actions/setup-backend/action.yml +++ b/.github/actions/setup-backend/action.yml @@ -2,9 +2,9 @@ name: 'Setup Python Environment' description: 'Set up Python and install dependencies with optional configurations.' inputs: python-version: - description: 'Python version to set up.' + description: 'Python version to set up. Accepts a version number, "current", or "next".' required: true - default: '3.10' + default: 'current' cache: description: 'Cache dependencies. Options: pip' required: false @@ -13,22 +13,39 @@ inputs: description: 'Type of requirements to install. Options: base, development, default' required: false default: 'dev' + install-superset: + description: 'Whether to install Superset itself. If false, only python is installed' + required: false + default: 'true' runs: using: 'composite' steps: - - name: Set up Python ${{ inputs.python-version }} + - name: Interpret Python Version + id: set-python-version + shell: bash + run: | + if [ "${{ inputs.python-version }}" = "current" ]; then + echo "PYTHON_VERSION=3.10" >> $GITHUB_ENV + elif [ "${{ inputs.python-version }}" = "next" ]; then + echo "PYTHON_VERSION=3.11" >> $GITHUB_ENV + else + echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV + fi + - name: Set up Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@v5 with: - python-version: ${{ inputs.python-version }} + python-version: ${{ env.PYTHON_VERSION }} cache: ${{ inputs.cache }} - name: Install dependencies run: | - sudo apt-get update && sudo apt-get -y install libldap2-dev libsasl2-dev - pip install --upgrade pip setuptools wheel - if [ "${{ inputs.requirements-type }}" = "dev" ]; then - pip install -r requirements/development.txt - elif [ "${{ inputs.requirements-type }}" = "base" ]; then - pip install -r requirements/base.txt + if [ "${{ inputs.install-superset }}" = "true" ]; then + sudo apt-get update && sudo apt-get -y install libldap2-dev libsasl2-dev + pip install --upgrade pip setuptools wheel + if [ "${{ inputs.requirements-type }}" = "dev" ]; then + pip install -r requirements/development.txt + elif [ "${{ inputs.requirements-type }}" = "base" ]; then + pip install -r requirements/base.txt + fi fi shell: bash diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 18001241ab83..e749ac35c812 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -73,6 +73,8 @@ jobs: - name: Build Docker Image if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | # Single platform builds in pull_request context to speed things up if [ "${{ github.event_name }}" = "push" ]; then diff --git a/.github/workflows/no-op.yml b/.github/workflows/no-op.yml new file mode 100644 index 000000000000..e56177239baf --- /dev/null +++ b/.github/workflows/no-op.yml @@ -0,0 +1,96 @@ +# no-op.yml +# +# Purpose: +# This workflow provides a workaround for the "required status checks" feature in GitHub Actions +# when using path-specific conditions in other workflows. Required checks might remain in a "Pending" +# state if the conditions are not met, thus blocking pull requests from being merged. +# This no-op (no operation) workflow provides dummy success statuses for these required jobs when +# the real jobs do not run due to path-specific conditions. +# +# How it works: +# - It defines jobs with the same names as the required jobs in the main workflows. +# - These jobs simply execute a command (`exit 0`) to succeed immediately. +# - When a pull request is created or updated, both this no-op workflow and the main workflows are triggered. +# - If the main workflows' jobs don't run (due to path conditions), these no-op jobs provide successful statuses. +# - If the main workflows' jobs do run and fail, their failure statuses take precedence, +# ensuring that pull requests are not merged with failing checks. +# +# Usage: +# - Ensure that the job names in this workflow match exactly the names of the corresponding jobs in the main workflows. +# - This workflow should be kept as-is, without path-specific conditions. + +name: no-op Checks +on: pull_request + +jobs: + frontend-build: + runs-on: ubuntu-latest + steps: + - name: No-op for frontend-build + run: exit 0 + + pre-commit: + strategy: + matrix: + python-version: ["3.10"] + runs-on: ubuntu-latest + steps: + - name: No-op for pre-commit + run: exit 0 + + python-lint: + strategy: + matrix: + python-version: ["3.10"] + runs-on: ubuntu-latest + steps: + - name: No-op for python-lint + run: exit 0 + test-postgres-hive: + strategy: + matrix: + python-version: ["3.10"] + runs-on: ubuntu-latest + steps: + - name: No-op for frontend-build + run: exit 0 + test-postgres-presto: + strategy: + matrix: + python-version: ["3.10"] + runs-on: ubuntu-latest + steps: + - name: No-op for frontend-build + run: exit 0 + unit-tests: + strategy: + matrix: + python-version: ["3.10"] + runs-on: ubuntu-latest + steps: + - name: No-op for frontend-build + run: exit 0 + test-mysql: + strategy: + matrix: + python-version: ["3.10"] + runs-on: ubuntu-latest + steps: + - name: No-op for test-mysql + run: exit 0 + test-postgres: + strategy: + matrix: + python-version: ["3.10", "3.11"] + runs-on: ubuntu-latest + steps: + - name: No-op for test-postgres + run: exit 0 + test-sqlite: + strategy: + matrix: + python-version: ["3.10"] + runs-on: ubuntu-latest + steps: + - name: No-op for test-sqlite + run: exit 0 diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 27af14d224d6..541b4a8bfc58 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -16,9 +16,6 @@ concurrency: jobs: pre-commit: runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ["3.10"] steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 diff --git a/.github/workflows/superset-cli.yml b/.github/workflows/superset-cli.yml index 1d9870a0b2c0..3c98198c7e73 100644 --- a/.github/workflows/superset-cli.yml +++ b/.github/workflows/superset-cli.yml @@ -16,9 +16,6 @@ concurrency: jobs: test-load-examples: runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ["3.10"] env: PYTHONPATH: ${{ github.workspace }} SUPERSET_CONFIG: tests.integration_tests.superset_test_config diff --git a/.github/workflows/superset-helm-lint.yml b/.github/workflows/superset-helm-lint.yml index f16dd61fdde3..f9e068318d2e 100644 --- a/.github/workflows/superset-helm-lint.yml +++ b/.github/workflows/superset-helm-lint.yml @@ -27,9 +27,10 @@ jobs: with: version: v3.5.4 - - uses: actions/setup-python@v5 + - name: Setup Python + uses: ./.github/actions/setup-backend/ with: - python-version: "3.10" + install-superset: 'false' - name: Set up chart-testing uses: ./.github/actions/chart-testing-action diff --git a/.github/workflows/superset-python-integrationtest.yml b/.github/workflows/superset-python-integrationtest.yml index 4d710d5bb590..2c1e721df479 100644 --- a/.github/workflows/superset-python-integrationtest.yml +++ b/.github/workflows/superset-python-integrationtest.yml @@ -16,9 +16,6 @@ concurrency: jobs: test-mysql: runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ["3.10"] env: PYTHONPATH: ${{ github.workspace }} SUPERSET_CONFIG: tests.integration_tests.superset_test_config @@ -51,14 +48,11 @@ jobs: - name: Setup Python uses: ./.github/actions/setup-backend/ if: steps.check.outputs.python - with: - python-version: ${{ matrix.python-version }} - name: Setup MySQL if: steps.check.outputs.python uses: ./.github/actions/cached-dependencies with: - run: | - setup-mysql + run: setup-mysql - name: Run celery if: steps.check.outputs.python run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 & @@ -74,7 +68,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.10", "3.11"] + python-version: ["current", "next"] env: PYTHONPATH: ${{ github.workspace }} SUPERSET_CONFIG: tests.integration_tests.superset_test_config @@ -130,9 +124,6 @@ jobs: test-sqlite: runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ["3.10"] env: PYTHONPATH: ${{ github.workspace }} SUPERSET_CONFIG: tests.integration_tests.superset_test_config @@ -160,8 +151,6 @@ jobs: - name: Setup Python uses: ./.github/actions/setup-backend/ if: steps.check.outputs.python - with: - python-version: ${{ matrix.python-version }} - name: Install dependencies if: steps.check.outputs.python uses: ./.github/actions/cached-dependencies diff --git a/.github/workflows/superset-python-misc.yml b/.github/workflows/superset-python-misc.yml index 2aacf8799be1..57a5b748515e 100644 --- a/.github/workflows/superset-python-misc.yml +++ b/.github/workflows/superset-python-misc.yml @@ -17,9 +17,6 @@ concurrency: jobs: python-lint: runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ["3.10"] steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 @@ -34,8 +31,6 @@ jobs: - name: Setup Python uses: ./.github/actions/setup-backend/ if: steps.check.outputs.python - with: - python-version: ${{ matrix.python-version }} - name: pylint if: steps.check.outputs.python # `-j 0` run Pylint in parallel @@ -43,9 +38,6 @@ jobs: babel-extract: runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ["3.10"] steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 @@ -60,8 +52,6 @@ jobs: - name: Setup Python if: steps.check.outputs.python uses: ./.github/actions/setup-backend/ - with: - python-version: ${{ matrix.python-version }} - name: Test babel extraction if: steps.check.outputs.python run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct diff --git a/.github/workflows/superset-python-presto-hive.yml b/.github/workflows/superset-python-presto-hive.yml index 66c6952c7d31..cf3e506bb9cc 100644 --- a/.github/workflows/superset-python-presto-hive.yml +++ b/.github/workflows/superset-python-presto-hive.yml @@ -17,9 +17,6 @@ concurrency: jobs: test-postgres-presto: runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ["3.10"] env: PYTHONPATH: ${{ github.workspace }} SUPERSET_CONFIG: tests.integration_tests.superset_test_config @@ -84,9 +81,6 @@ jobs: test-postgres-hive: runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ["3.10"] env: PYTHONPATH: ${{ github.workspace }} SUPERSET_CONFIG: tests.integration_tests.superset_test_config diff --git a/.github/workflows/superset-python-unittest.yml b/.github/workflows/superset-python-unittest.yml index 48e2c21c2637..cf3e7c764118 100644 --- a/.github/workflows/superset-python-unittest.yml +++ b/.github/workflows/superset-python-unittest.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.10", "3.11"] + python-version: ["current", "next"] env: PYTHONPATH: ${{ github.workspace }} steps: diff --git a/.github/workflows/superset-translations.yml b/.github/workflows/superset-translations.yml index 734e9bf752ab..42cf76cdf19d 100644 --- a/.github/workflows/superset-translations.yml +++ b/.github/workflows/superset-translations.yml @@ -47,9 +47,6 @@ jobs: babel-extract: runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ["3.10"] steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v4 @@ -65,8 +62,6 @@ jobs: - name: Setup Python if: steps.check.outputs.python uses: ./.github/actions/setup-backend/ - with: - python-version: ${{ matrix.python-version }} - name: Test babel extraction if: steps.check.outputs.python run: ./scripts/babel_update.sh