From 956add93928b871b2d1caa483d3bb0630802f0e4 Mon Sep 17 00:00:00 2001 From: Ekleog-NEAR <96595974+Ekleog-NEAR@users.noreply.github.com> Date: Mon, 20 Nov 2023 12:53:43 +0100 Subject: [PATCH] introduce code coverage analysis in CI (#10180) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Note: I’m removing the preparatory "Build neard" job here. The reason is twofold: - I didn’t manage to get coverage output to actually write *.profraw files without the whole target folder - Uploading the whole compressed (to 3G) target folder as an artifact takes 6 minutes for compression and 15 minutes for upload - Just building, on the commit on which I was, took 2 minutes, multiplied by 4 it gives 8 minutes, so quite a lot less than the compress+upload time - So overall we’re spending less time by just redoing the compilation work, as opposed to uploading+downloading it; and the tests themselves are relatively fast too so having them on the bigger machines (8cores, enough for them to not be the bottleneck) shouldn’t hog them for too long. --- .github/workflows/ci.yml | 196 ++++++++++++++++++++++++++------------- 1 file changed, 132 insertions(+), 64 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7a5c132a3a..b3ac41a341a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,24 +9,6 @@ on: merge_group: jobs: - build_binary: - name: "Build neard" - runs-on: ubuntu-22.04-16core - steps: - - uses: actions/checkout@v4 - - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43 - with: - prefix-key: "0" # change this to invalidate CI cache - shared-key: "cargo_nextest-linux" - save-if: "false" # use the cache from nextest, but don’t double-save - - run: cargo build --locked --profile dev-release -p neard --bin neard - - uses: actions/upload-artifact@v3 - with: - name: neard - path: target/dev-release/neard - if-no-files-found: error - retention-days: 1 - cargo_nextest: name: "Cargo Nextest (${{matrix.name}})" runs-on: ${{ matrix.os }} @@ -35,39 +17,79 @@ jobs: matrix: include: - name: Linux + id: linux cache_id: linux os: ubuntu-22.04-16core flags: "" + run_integ_tests: true - name: Linux Nightly + id: linux-nightly cache_id: linux os: ubuntu-22.04-16core flags: "--features nightly,test_features" + run_integ_tests: true - name: MacOS + id: macos cache_id: macos os: macos-latest-xlarge # FIXME: some of these tests don't work very well on MacOS at the moment. Should fix # them at earliest convenience :) - flags: "--exclude integration-tests --exclude node-runtime --exclude runtime-params-estimator --exclude near-network --exclude estimator-warehouse" + flags: "--exclude node-runtime --exclude runtime-params-estimator --exclude near-network --exclude estimator-warehouse" + run_integ_tests: false timeout-minutes: 90 steps: - uses: actions/checkout@v4 + + # Install all the required tools - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941 with: crate: cargo-nextest - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941 with: crate: cargo-deny + - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941 + with: + crate: cargo-llvm-cov + + # Setup the dependency rust cache and llvm-cov - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43 with: prefix-key: "0" # change this to invalidate CI cache shared-key: "cargo_nextest-${{ matrix.cache_id }}" - - run: cargo nextest run --locked --workspace -p '*' --cargo-profile dev-release --profile ci ${{ matrix.flags }} + - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV" + + # Run unit tests + - run: cargo nextest run --locked --workspace --exclude integration-tests --cargo-profile dev-release --profile ci ${{ matrix.flags }} env: RUST_BACKTRACE: short + - run: cargo llvm-cov report --profile dev-release --codecov --output-path unittests.json + - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: unittests.json + fail_ci_if_error: true + flags: unittests,${{ matrix.id }} + # See https://github.com/taiki-e/cargo-llvm-cov/issues/292 + - run: find target -name '*.profraw' -delete + + # Run integration tests + - run: cargo nextest run --locked --package integration-tests --cargo-profile dev-release --profile ci ${{ matrix.flags }} + if: matrix.run_integ_tests + env: + RUST_BACKTRACE: short + - run: cargo llvm-cov report --profile dev-release --codecov --output-path integration-tests.json + if: matrix.run_integ_tests + - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d + if: matrix.run_integ_tests + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: integration-tests.json + fail_ci_if_error: true + flags: integration-tests,${{ matrix.id }} protobuf_backward_compat: name: "Protobuf Backward Compatibility" - runs-on: ubuntu-latest + runs-on: ubuntu-22.04-8core steps: - uses: actions/checkout@v4 - uses: bufbuild/buf-setup-action@1158f4fa81bc02e1ff62abcca6d516c9e24c77da @@ -77,48 +99,64 @@ jobs: py_backward_compat: name: "Backward Compatibility" - needs: build_binary runs-on: ubuntu-22.04 - defaults: - run: - working-directory: ./pytest steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: 3.11 cache: pip - - uses: actions/download-artifact@v3 + - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941 + with: + crate: cargo-llvm-cov + - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43 + with: + prefix-key: "0" # change this to invalidate CI cache + shared-key: "cargo_nextest-linux" + save-if: "false" # use the cache from nextest, but don’t double-save + - run: pip3 install --user -r pytest/requirements.txt + - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV" + - run: cargo build --locked --profile dev-release -p neard --bin neard + - run: echo "CURRENT_NEARD=$PWD/target/dev-release/neard" >> "$GITHUB_ENV" + - run: cd pytest && python3 tests/sanity/backward_compatible.py + - run: cargo llvm-cov report --profile dev-release --codecov --output-path pytest-backcomp.json + - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d with: - name: neard - path: pytest # NB: this does not account for defaults.run.working-directory - - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV" - - run: chmod +x "$CURRENT_NEARD" - - run: pip3 install --user -r requirements.txt - - run: python3 tests/sanity/backward_compatible.py + token: ${{ secrets.CODECOV_TOKEN }} + files: pytest-backcomp.json + fail_ci_if_error: true + flags: pytests,backward-compatibility,linux py_db_migration: name: "Database Migration" - needs: build_binary - runs-on: ubuntu-22.04 - defaults: - run: - working-directory: ./pytest + runs-on: ubuntu-22.04-8core steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: 3.11 cache: pip - - uses: actions/download-artifact@v3 + - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941 with: - name: neard - path: pytest # NB: this does not account for defaults.run.working-directory - - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV" + crate: cargo-llvm-cov + - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43 + with: + prefix-key: "0" # change this to invalidate CI cache + shared-key: "cargo_nextest-linux" + save-if: "false" # use the cache from nextest, but don’t double-save + - run: pip3 install --user -r pytest/requirements.txt + - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV" + - run: cargo build --locked --profile dev-release -p neard --bin neard + - run: echo "CURRENT_NEARD=$PWD/target/dev-release/neard" >> "$GITHUB_ENV" - run: echo "NEAR_ROOT=$PWD" >> "$GITHUB_ENV" - - run: chmod +x "$CURRENT_NEARD" - - run: pip3 install --user -r requirements.txt - - run: python3 tests/sanity/db_migration.py + - run: cd pytest && python3 tests/sanity/db_migration.py + - run: cargo llvm-cov report --profile dev-release --codecov --output-path pytest-dbmigr.json + - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: pytest-dbmigr.json + fail_ci_if_error: true + flags: pytests,db-migration,linux py_sanity_checks: name: "Sanity Checks" @@ -132,6 +170,9 @@ jobs: with: python-version: 3.11 cache: pip + - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941 + with: + crate: cargo-llvm-cov - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43 with: prefix-key: "0" # change this to invalidate CI cache @@ -140,6 +181,7 @@ jobs: - run: pip3 install --user -r pytest/requirements.txt # This is the only job that uses `--features nightly` so we build this in-line instead of a # separate job like done with the regular neard. + - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV" - run: cargo build --profile dev-release -p neard --bin neard --features nightly # Note: We're not running spin_up_cluster.py for non-nightly # because spinning up non-nightly clusters is already covered @@ -147,25 +189,43 @@ jobs: - run: python3 pytest/tests/sanity/spin_up_cluster.py env: NEAR_ROOT: "target/dev-release" + - run: cargo llvm-cov report --profile dev-release --codecov --output-path pytest-sanity.json + - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: pytest-sanity.json + fail_ci_if_error: true + flags: pytests,sanity-checks,linux-nightly py_genesis_check: name: "Genesis Changes" - needs: build_binary - runs-on: ubuntu-22.04 + runs-on: ubuntu-22.04-8core steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: 3.11 cache: pip - - uses: actions/download-artifact@v3 + - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941 with: - name: neard - path: target/dev-release - - run: echo "CURRENT_NEARD=$PWD/target/dev-release/neard" >> "$GITHUB_ENV" - - run: chmod +x "$CURRENT_NEARD" + crate: cargo-llvm-cov + - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43 + with: + prefix-key: "0" # change this to invalidate CI cache + shared-key: "cargo_nextest-linux" + save-if: "false" # use the cache from nextest, but don’t double-save - run: pip3 install --user -r pytest/requirements.txt + - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV" + - run: cargo build --locked --profile dev-release -p neard --bin neard + - run: echo "CURRENT_NEARD=$PWD/target/dev-release/neard" >> "$GITHUB_ENV" - run: python3 scripts/state/update_res.py check + - run: cargo llvm-cov report --profile dev-release --codecov --output-path pytest-genesischk.json + - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: pytest-genesischk.json + fail_ci_if_error: true + flags: pytests,genesis-check,linux py_style_check: name: "Style" @@ -184,25 +244,33 @@ jobs: py_upgradability: name: "Upgradability" - needs: build_binary - runs-on: ubuntu-22.04 - defaults: - run: - working-directory: ./pytest + runs-on: ubuntu-22.04-8core steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: 3.11 cache: pip - - uses: actions/download-artifact@v3 - with: - name: neard - path: pytest # NB: this does not account for defaults.run.working-directory - - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV" - - run: chmod +x "$CURRENT_NEARD" - - run: pip3 install --user -r requirements.txt - - run: python3 tests/sanity/upgradable.py + - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941 + with: + crate: cargo-llvm-cov + - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43 + with: + prefix-key: "0" # change this to invalidate CI cache + shared-key: "cargo_nextest-linux" + save-if: "false" # use the cache from nextest, but don’t double-save + - run: pip3 install --user -r pytest/requirements.txt + - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV" + - run: cargo build --locked --profile dev-release -p neard --bin neard + - run: echo "CURRENT_NEARD=$PWD/target/dev-release/neard" >> "$GITHUB_ENV" + - run: cd pytest && python3 tests/sanity/upgradable.py + - run: cargo llvm-cov report --profile dev-release --codecov --output-path pytest-upgradability.json + - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: pytest-upgradability.json + fail_ci_if_error: true + flags: pytests,upgradability,linux rpc_error_schema: name: "RPC Schema"