Refactor build process #5
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: main | |
| on: | |
| push: | |
| pull_request: | |
| concurrency: | |
| group: main-${{ github.head_ref }} | |
| cancel-in-progress: true | |
| env: | |
| FORCE_COLOR: 3 | |
| jobs: | |
| get_python_version: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| PYVERSION: ${{ steps.get_python_version.outputs.PYVERSION }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Read the Python version from Makefile.envs | |
| id: get_python_version | |
| run: | | |
| echo "PYVERSION=$(git grep 'export PYVERSION ?=' Makefile.envs | cut -d' ' -f4)" >> "$GITHUB_OUTPUT" | |
| test-python: | |
| runs-on: ubuntu-latest | |
| needs: get_python_version | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ needs.get_python_version.outputs.PYVERSION }} | |
| - name: Install requirements | |
| shell: bash -l {0} | |
| run: | | |
| mkdir test-results | |
| python3 -m pip install -r requirements.txt -r requirements-deploy.txt | |
| make pyodide_build | |
| - name: Run tests | |
| shell: bash -l {0} | |
| run: | | |
| PYODIDE_ROOT=. pytest \ | |
| --junitxml=test-results/junit.xml \ | |
| --verbose \ | |
| --runtime=host \ | |
| --cov=pyodide_build --cov=pyodide \ | |
| src packages/_tests tools/ | |
| - uses: codecov/codecov-action@v4 | |
| with: | |
| fail_ci_if_error: false | |
| build-core: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: ubuntu-latest | |
| pyodide_packages: "tag:core,numpy${{ needs.test-scipy-trigger.outputs.test-scipy == 'true' && ',scipy' || '' }}" | |
| - os: macos-14 | |
| pyodide_packages: "tag:core,numpy" | |
| runs-on: ${{ matrix.os }} | |
| needs: [test-scipy-trigger] | |
| env: | |
| EMSDK_NUM_CORES: 3 | |
| EMCC_CORES: 3 | |
| PYODIDE_JOBS: 3 | |
| CCACHE_DIR: /tmp/ccache | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Cache ccache output | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| /tmp/ccache | |
| key: ${{ hashFiles('Makefile.envs') }}-${{ runner.os }}-v20211025- | |
| - uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| activate-environment: pyodide-env | |
| environment-file: environment.yml | |
| channels: conda-forge | |
| - name: Check Versions | |
| shell: bash -l {0} | |
| run: | | |
| python --version | |
| node --version | |
| npm --version | |
| which python | |
| which node | |
| which npm | |
| - name: Install dependencies (ubuntu) | |
| shell: bash -l {0} | |
| if: ${{ contains(runner.os, 'ubuntu') }} | |
| run: | | |
| sudo apt install -y build-essential git xxd | |
| - name: Install dependencies (macos) | |
| shell: bash -l {0} | |
| if: ${{ contains(runner.os, 'macos') }} | |
| run: | | |
| brew install coreutils | |
| - name: Install dependencies (Python) | |
| shell: bash -l {0} | |
| run: | | |
| pip install -r requirements.txt | |
| - name: Build emsdk | |
| shell: bash -l {0} | |
| run: | | |
| which ccache | |
| ccache -z | |
| make -C emsdk | |
| ccache -s | |
| - name: Build Cpython | |
| shell: bash -l {0} | |
| run: | | |
| # This is necessary to use the ccache from emsdk | |
| source pyodide_env.sh | |
| which ccache | |
| ccache -z | |
| make -C cpython | |
| ccache -s | |
| - name: build Pyodide with packages ${{ matrix.pyodide_packages }} | |
| shell: bash -l {0} | |
| run: | | |
| # This is necessary to use the ccache from emsdk | |
| source pyodide_env.sh | |
| ccache -z | |
| PYODIDE_PACKAGES=${{ matrix.pyodide_packages }} make | |
| ccache -s | |
| - name: check-size | |
| shell: bash -l {0} | |
| run: | | |
| ls -lh dist/ | |
| pip install brotli | |
| ./tools/check_compressed_size.py dist/pyodide.asm.* dist/python_stdlib* | |
| - name: Store artifacts build | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: core-build-${{ runner.os }} | |
| path: ./dist/ | |
| retention-days: 60 | |
| test-core: | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| DISPLAY: :99 | |
| needs: [build-core] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [macos-14] | |
| runner: [selenium] | |
| browser: [safari] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download build artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: core-build-${{ runner.os }} | |
| path: ./dist/ | |
| - uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| activate-environment: pyodide-env | |
| python-version: "3.12" | |
| channels: conda-forge | |
| - name: install test requirements | |
| shell: bash -l {0} | |
| run: | | |
| pip install -r requirements.txt -r requirements-deploy.txt | |
| make pyodide_build | |
| - uses: pyodide/pyodide-actions/install-browser@v2 | |
| with: | |
| runner: ${{ matrix.runner }} | |
| browser: ${{ matrix.browser }} | |
| - name: run core tests | |
| env: | |
| BROWSER: ${{ matrix.browser }} | |
| RUNNER: ${{ matrix.runner }} | |
| shell: bash -l {0} | |
| run: | | |
| ls -lh | |
| ls -lh dist/ | |
| tools/pytest_wrapper.py src packages/micropip/ tools/ \ | |
| -v \ | |
| -k "not webworker" \ | |
| --runtime="${BROWSER}-no-host" \ | |
| --runner "${RUNNER}" \ | |
| --durations 50 \ | |
| --junitxml=test-results/core_test.xml | |
| - name: run package tests | |
| env: | |
| BROWSER: ${{ matrix.browser }} | |
| RUNNER: ${{ matrix.runner }} | |
| shell: bash -l {0} | |
| run: | | |
| ls -lh | |
| ls -lh dist/ | |
| tools/pytest_wrapper.py packages/*/test* \ | |
| -v \ | |
| -k "numpy and not joblib" \ | |
| --runtime="${BROWSER}-no-host" \ | |
| --runner "${RUNNER}" \ | |
| --durations 50 \ | |
| --junitxml=test-results/packages_test.xml | |
| - name: Test Summary | |
| uses: test-summary/action@v2 | |
| with: | |
| paths: "test-results/*.xml" | |
| if: always() | |
| test-scipy-trigger: | |
| name: test-scipy-trigger | |
| runs-on: ubuntu-latest | |
| outputs: | |
| test-scipy: ${{ steps.check-build-trigger.outputs.trigger }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Check build trigger | |
| id: check-build-trigger | |
| shell: bash | |
| run: | | |
| set -e -x | |
| COMMIT_MSG=$(git log --no-merges -1 --oneline) | |
| # The scipy tests will be triggered on push or on pull_request when the commit | |
| # message contains "[scipy]" | |
| if [[ "$GITHUB_EVENT_NAME" == push || | |
| "$COMMIT_MSG" =~ \[scipy\] ]]; then | |
| echo "trigger=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| test-scipy: | |
| runs-on: ${{ matrix.os }} | |
| needs: [test-scipy-trigger, build-core] | |
| if: needs.test-scipy-trigger.outputs.test-scipy | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download build artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: core-build-${{ runner.os }} | |
| path: ./dist/ | |
| - name: run scipy tests inside node | |
| shell: bash -l {0} | |
| run: | | |
| npm install pyodide | |
| cp -f dist/* node_modules/pyodide | |
| # This uses conftest.py on the current working directory to | |
| # skip/xfail tests in scipy | |
| cd packages/scipy | |
| # XXX for some unknown reason adding a conftest.py in the repo throws off the | |
| # other tests trying to import from conftest they find the scipy one ... | |
| mv scipy-conftest.py conftest.py | |
| node scipy-pytest.js --pyargs scipy -m 'not slow' -ra -v | |
| test-bun: | |
| runs-on: ${{ matrix.os }} | |
| needs: [build-core] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download build artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: core-build-${{ runner.os }} | |
| path: ./dist/ | |
| - uses: oven-sh/setup-bun@v2 | |
| - name: install test requirements | |
| working-directory: src/test-bun | |
| run: | | |
| bun --version | |
| bun install | |
| - name: run bun tests | |
| working-directory: src/test-bun | |
| run: bun smoke-test.js | |
| test-deno: | |
| runs-on: ${{ matrix.os }} | |
| needs: [build-core] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download build artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: core-build-${{ runner.os }} | |
| path: ./dist/ | |
| - name: install test requirements | |
| shell: bash -l {0} | |
| run: | | |
| curl -fsSL https://deno.land/install.sh | sudo DENO_INSTALL=/usr/local sh -s v1.33.1 | |
| deno --version | |
| - name: run deno tests | |
| shell: bash -l {0} | |
| run: cd src/test-deno && deno task test |