From ee7b9e5b4468e9d07137569fb872380a204540f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Wed, 15 Nov 2023 13:11:47 +0000
Subject: [PATCH 01/24] introduce code coverage analysis in CI

---
 .github/workflows/ci.yml | 100 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 99 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d35f972db2e..5f516f29323 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -8,17 +8,27 @@ on:
   pull_request:
   merge_group:
 
+env:
+  CARGO_TERM_COLOR: always
+
 jobs:
   build_binary:
     name: "Build neard"
     runs-on: ubuntu-22.04-16core
     steps:
       - uses: actions/checkout@v4
+      - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
+        with:
+          crate: cargo-llvm-cov
       - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
         with:
           prefix-key: "0" # change this to invalidate CI cache
           shared-key: "cargo_nextest-linux"
           save-if: "false" # use the cache from nextest, but don’t double-save
+      # TODO: remove the LLVM_PROFILE_FILE override once
+      # https://github.com/taiki-e/cargo-llvm-cov/issues/320 is fixed (and same below)
+      - run: cargo llvm-cov show-env --export-prefix >> "$GITHUB_ENV"
+      - run: echo "LLVM_PROFILE_FILE=$(pwd)/target/llvm-cov-target/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
       - run: cargo build --locked --profile quick-release -p neard --bin neard
       - uses: actions/upload-artifact@v3
         with:
@@ -35,14 +45,17 @@ jobs:
       matrix:
         include:
           - name: Linux
+            id: linux
             cache_id: linux
             os: ubuntu-22.04-16core
             flags: ""
           - name: Linux Nightly
+            id: linux-nightly
             cache_id: linux
             os: ubuntu-22.04-16core
             flags: "--features nightly,test_features"
           - name: MacOS
+            id: macos
             cache_id: macos
             os: macos-latest-xlarge
             # FIXME: some of these tests don't work very well on MacOS at the moment. Should fix
@@ -55,19 +68,49 @@ jobs:
       # FIXME(#9634): remove this once the issue is resolved.
       - run: sudo sysctl vm.overcommit_memory=1 || true
       - uses: actions/checkout@v4
+
+      # Install all the required tools
       - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
         with:
           crate: cargo-nextest
       - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
         with:
           crate: cargo-deny
+      - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
+        with:
+          crate: cargo-llvm-cov
+
+      # Setup the dependency rust cache and llvm-cov
       - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
         with:
           prefix-key: "0" # change this to invalidate CI cache
           shared-key: "cargo_nextest-${{ matrix.cache_id }}"
-      - run: cargo nextest run --locked --workspace -p '*' --cargo-profile quick-release --profile ci ${{ matrix.flags }}
+      - run: cargo llvm-cov show-env --export-prefix >> "$GITHUB_ENV"
+      - run: echo "LLVM_PROFILE_FILE=$(pwd)/target/llvm-cov-target/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
+
+      # Run unit tests
+      - run: cargo nextest run --locked --workspace --exclude integration-tests --cargo-profile quick-release --profile ci ${{ matrix.flags }}
         env:
           RUST_BACKTRACE: short
+      - run: cargo llvm-cov report --codecov --output-path unittests.json
+      - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
+        with:
+          files: unittests.json
+          fail_ci_if_error: true
+          flags: unittests,${{ matrix.id }}
+      # See https://github.com/taiki-e/cargo-llvm-cov/issues/292
+      - run: rm -f target/llvm-cov-target/*.profraw
+
+      # Run integration tests
+      - run: cargo nextest run --locked --package integration-tests --cargo-profile quick-release --profile ci ${{ matrix.flags }}
+        env:
+          RUST_BACKTRACE: short
+      - run: cargo llvm-cov report --codecov --output-path integration-tests.json
+      - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
+        with:
+          files: integration-tests.json
+          fail_ci_if_error: true
+          flags: integration-tests,${{ matrix.id }}
 
   protobuf_backward_compat:
     name: "Protobuf Backward Compatibility"
@@ -92,6 +135,9 @@ jobs:
         with:
           python-version: 3.11
           cache: pip
+      - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
+        with:
+          crate: cargo-llvm-cov
       - uses: actions/download-artifact@v3
         with:
           name: neard
@@ -99,7 +145,15 @@ jobs:
       - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r requirements.txt
+      - run: cargo llvm-cov show-env --export-prefix >> "$GITHUB_ENV"
+      - run: echo "LLVM_PROFILE_FILE=$(pwd)/../target/llvm-cov-target/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
       - run: python3 tests/sanity/backward_compatible.py
+      - run: cargo llvm-cov report --codecov --output-path pytest-backcomp.json
+      - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
+        with:
+          files: pytest-backcomp.json
+          fail_ci_if_error: true
+          flags: pytests,backward-compatibility,linux
 
   py_db_migration:
     name: "Database Migration"
@@ -114,6 +168,9 @@ jobs:
         with:
           python-version: 3.11
           cache: pip
+      - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
+        with:
+          crate: cargo-llvm-cov
       - uses: actions/download-artifact@v3
         with:
           name: neard
@@ -122,7 +179,15 @@ jobs:
       - run: echo "NEAR_ROOT=$PWD" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r requirements.txt
+      - run: cargo llvm-cov show-env --export-prefix >> "$GITHUB_ENV"
+      - run: echo "LLVM_PROFILE_FILE=$(pwd)/../target/llvm-cov-target/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
       - run: python3 tests/sanity/db_migration.py
+      - run: cargo llvm-cov report --codecov --output-path pytest-dbmigr.json
+      - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
+        with:
+          files: pytest-dbmigr.json
+          fail_ci_if_error: true
+          flags: pytests,db-migration,linux
 
   py_sanity_checks:
     name: "Sanity Checks"
@@ -136,6 +201,9 @@ jobs:
         with:
           python-version: 3.11
           cache: pip
+      - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
+        with:
+          crate: cargo-llvm-cov
       - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
         with:
           prefix-key: "0" # change this to invalidate CI cache
@@ -144,6 +212,8 @@ jobs:
       - run: pip3 install --user -r pytest/requirements.txt
       # This is the only job that uses `--features nightly` so we build this in-line instead of a
       # separate job like done with the regular neard.
+      - run: cargo llvm-cov show-env --export-prefix >> "$GITHUB_ENV"
+      - run: echo "LLVM_PROFILE_FILE=$(pwd)/target/llvm-cov-target/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
       - run: cargo build --profile quick-release -p neard --bin neard --features nightly
       # Note: We're not running spin_up_cluster.py for non-nightly
       # because spinning up non-nightly clusters is already covered
@@ -151,6 +221,12 @@ jobs:
       - run: python3 pytest/tests/sanity/spin_up_cluster.py
         env:
           NEAR_ROOT: "target/quick-release"
+      - run: cargo llvm-cov report --codecov --output-path pytest-sanity.json
+      - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
+        with:
+          files: pytest-sanity.json
+          fail_ci_if_error: true
+          flags: pytests,sanity-checks,linux-nightly
 
   py_genesis_check:
     name: "Genesis Changes"
@@ -162,6 +238,9 @@ jobs:
         with:
           python-version: 3.11
           cache: pip
+      - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
+        with:
+          crate: cargo-llvm-cov
       - uses: actions/download-artifact@v3
         with:
           name: neard
@@ -169,7 +248,15 @@ jobs:
       - run: echo "CURRENT_NEARD=$PWD/target/quick-release/neard" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r pytest/requirements.txt
+      - run: cargo llvm-cov show-env --export-prefix >> "$GITHUB_ENV"
+      - run: echo "LLVM_PROFILE_FILE=$(pwd)/target/llvm-cov-target/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
       - run: python3 scripts/state/update_res.py check
+      - run: cargo llvm-cov report --codecov --output-path pytest-genesischk.json
+      - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
+        with:
+          files: pytest-genesischk.json
+          fail_ci_if_error: true
+          flags: pytests,genesis-check,linux
 
   py_style_check:
     name: "Style"
@@ -199,6 +286,9 @@ jobs:
         with:
           python-version: 3.11
           cache: pip
+      - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
+        with:
+          crate: cargo-llvm-cov
       - uses: actions/download-artifact@v3
         with:
           name: neard
@@ -206,7 +296,15 @@ jobs:
       - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r requirements.txt
+      - run: cargo llvm-cov show-env --export-prefix >> "$GITHUB_ENV"
+      - run: echo "LLVM_PROFILE_FILE=$(pwd)/target/llvm-cov-target/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
       - run: python3 tests/sanity/upgradable.py
+      - run: cargo llvm-cov report --codecov --output-path pytest-upgradability.json
+      - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
+        with:
+          files: pytest-upgradability.json
+          fail_ci_if_error: true
+          flags: pytests,upgradability,linux
 
   rpc_error_schema:
     name: "RPC Schema"

From f35de0eca170f7fe9f0a2c7e1073c61ecb48369a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Wed, 15 Nov 2023 14:22:49 +0000
Subject: [PATCH 02/24] =?UTF-8?q?try=20workspace=20cleaning=20in=20case=20?=
 =?UTF-8?q?it=E2=80=99s=20the=20thing=20breaking=20it=20all?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .github/workflows/ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5f516f29323..679b4146b34 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -87,11 +87,13 @@ jobs:
           shared-key: "cargo_nextest-${{ matrix.cache_id }}"
       - run: cargo llvm-cov show-env --export-prefix >> "$GITHUB_ENV"
       - run: echo "LLVM_PROFILE_FILE=$(pwd)/target/llvm-cov-target/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
+      - run: cargo llvm-cov clean --workspace
 
       # Run unit tests
       - run: cargo nextest run --locked --workspace --exclude integration-tests --cargo-profile quick-release --profile ci ${{ matrix.flags }}
         env:
           RUST_BACKTRACE: short
+      - run: find . -name '*.profraw'
       - run: cargo llvm-cov report --codecov --output-path unittests.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:

From 82ffa9fd71708a941ab34027b6c16f0c34bb34df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Wed, 15 Nov 2023 14:59:57 +0000
Subject: [PATCH 03/24] try removing llvm_profile_file from nextest job

---
 .github/workflows/ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 679b4146b34..bba2b58f49f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -86,7 +86,6 @@ jobs:
           prefix-key: "0" # change this to invalidate CI cache
           shared-key: "cargo_nextest-${{ matrix.cache_id }}"
       - run: cargo llvm-cov show-env --export-prefix >> "$GITHUB_ENV"
-      - run: echo "LLVM_PROFILE_FILE=$(pwd)/target/llvm-cov-target/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
       - run: cargo llvm-cov clean --workspace
 
       # Run unit tests

From 5c21b418b7a7266f5983f4ffbfb7234fd7f2e64b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Wed, 15 Nov 2023 15:07:38 +0000
Subject: [PATCH 04/24] remove export-prefix

---
 .github/workflows/ci.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index bba2b58f49f..c5769bba776 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -27,7 +27,7 @@ jobs:
           save-if: "false" # use the cache from nextest, but don’t double-save
       # TODO: remove the LLVM_PROFILE_FILE override once
       # https://github.com/taiki-e/cargo-llvm-cov/issues/320 is fixed (and same below)
-      - run: cargo llvm-cov show-env --export-prefix >> "$GITHUB_ENV"
+      - run: cargo llvm-cov show-env >> "$GITHUB_ENV"
       - run: echo "LLVM_PROFILE_FILE=$(pwd)/target/llvm-cov-target/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
       - run: cargo build --locked --profile quick-release -p neard --bin neard
       - uses: actions/upload-artifact@v3
@@ -85,7 +85,7 @@ jobs:
         with:
           prefix-key: "0" # change this to invalidate CI cache
           shared-key: "cargo_nextest-${{ matrix.cache_id }}"
-      - run: cargo llvm-cov show-env --export-prefix >> "$GITHUB_ENV"
+      - run: cargo llvm-cov show-env >> "$GITHUB_ENV"
       - run: cargo llvm-cov clean --workspace
 
       # Run unit tests
@@ -146,7 +146,7 @@ jobs:
       - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r requirements.txt
-      - run: cargo llvm-cov show-env --export-prefix >> "$GITHUB_ENV"
+      - run: cargo llvm-cov show-env >> "$GITHUB_ENV"
       - run: echo "LLVM_PROFILE_FILE=$(pwd)/../target/llvm-cov-target/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
       - run: python3 tests/sanity/backward_compatible.py
       - run: cargo llvm-cov report --codecov --output-path pytest-backcomp.json
@@ -180,7 +180,7 @@ jobs:
       - run: echo "NEAR_ROOT=$PWD" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r requirements.txt
-      - run: cargo llvm-cov show-env --export-prefix >> "$GITHUB_ENV"
+      - run: cargo llvm-cov show-env >> "$GITHUB_ENV"
       - run: echo "LLVM_PROFILE_FILE=$(pwd)/../target/llvm-cov-target/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
       - run: python3 tests/sanity/db_migration.py
       - run: cargo llvm-cov report --codecov --output-path pytest-dbmigr.json
@@ -213,7 +213,7 @@ jobs:
       - run: pip3 install --user -r pytest/requirements.txt
       # This is the only job that uses `--features nightly` so we build this in-line instead of a
       # separate job like done with the regular neard.
-      - run: cargo llvm-cov show-env --export-prefix >> "$GITHUB_ENV"
+      - run: cargo llvm-cov show-env >> "$GITHUB_ENV"
       - run: echo "LLVM_PROFILE_FILE=$(pwd)/target/llvm-cov-target/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
       - run: cargo build --profile quick-release -p neard --bin neard --features nightly
       # Note: We're not running spin_up_cluster.py for non-nightly
@@ -249,7 +249,7 @@ jobs:
       - run: echo "CURRENT_NEARD=$PWD/target/quick-release/neard" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r pytest/requirements.txt
-      - run: cargo llvm-cov show-env --export-prefix >> "$GITHUB_ENV"
+      - run: cargo llvm-cov show-env >> "$GITHUB_ENV"
       - run: echo "LLVM_PROFILE_FILE=$(pwd)/target/llvm-cov-target/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
       - run: python3 scripts/state/update_res.py check
       - run: cargo llvm-cov report --codecov --output-path pytest-genesischk.json
@@ -297,7 +297,7 @@ jobs:
       - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r requirements.txt
-      - run: cargo llvm-cov show-env --export-prefix >> "$GITHUB_ENV"
+      - run: cargo llvm-cov show-env >> "$GITHUB_ENV"
       - run: echo "LLVM_PROFILE_FILE=$(pwd)/target/llvm-cov-target/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
       - run: python3 tests/sanity/upgradable.py
       - run: cargo llvm-cov report --codecov --output-path pytest-upgradability.json

From 4653fd50024048bc4c2460f7b3fb0df01f2c4939 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Wed, 15 Nov 2023 15:13:19 +0000
Subject: [PATCH 05/24] try with proper github_env syntax

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c5769bba776..3813da6b057 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -85,7 +85,7 @@ jobs:
         with:
           prefix-key: "0" # change this to invalidate CI cache
           shared-key: "cargo_nextest-${{ matrix.cache_id }}"
-      - run: cargo llvm-cov show-env >> "$GITHUB_ENV"
+      - run: source <(cargo llvm-cov show-env) && env > "$GITHUB_ENV"
       - run: cargo llvm-cov clean --workspace
 
       # Run unit tests

From 2a739fd5226fcb1cc6656940e7c8c89518de15e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Wed, 15 Nov 2023 15:41:53 +0000
Subject: [PATCH 06/24] use simpler way of setting the env

---
 .github/workflows/ci.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3813da6b057..85683350d52 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -85,8 +85,7 @@ jobs:
         with:
           prefix-key: "0" # change this to invalidate CI cache
           shared-key: "cargo_nextest-${{ matrix.cache_id }}"
-      - run: source <(cargo llvm-cov show-env) && env > "$GITHUB_ENV"
-      - run: cargo llvm-cov clean --workspace
+      - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
 
       # Run unit tests
       - run: cargo nextest run --locked --workspace --exclude integration-tests --cargo-profile quick-release --profile ci ${{ matrix.flags }}

From bd4812a102c6ec62a0dc0acb40049b0707a36a71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Wed, 15 Nov 2023 15:42:32 +0000
Subject: [PATCH 07/24] add debug target listing

---
 .github/workflows/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 85683350d52..478985923cb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -92,6 +92,7 @@ jobs:
         env:
           RUST_BACKTRACE: short
       - run: find . -name '*.profraw'
+      - run: find target
       - run: cargo llvm-cov report --codecov --output-path unittests.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:

From d1ad35f3e1acef5ace4017d70d022546e235e75e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Wed, 15 Nov 2023 16:09:50 +0000
Subject: [PATCH 08/24] try with llvm profile file override again

---
 .github/workflows/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 478985923cb..570c67a8140 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -86,6 +86,7 @@ jobs:
           prefix-key: "0" # change this to invalidate CI cache
           shared-key: "cargo_nextest-${{ matrix.cache_id }}"
       - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
+      - run: echo "LLVM_PROFILE_FILE=$(pwd)/target/debug/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
 
       # Run unit tests
       - run: cargo nextest run --locked --workspace --exclude integration-tests --cargo-profile quick-release --profile ci ${{ matrix.flags }}

From 3ff8d467a9c6c32e2375143a7bc0571891feea54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Thu, 16 Nov 2023 01:18:42 +0000
Subject: [PATCH 09/24] remove llvm_profile_file trick

---
 .github/workflows/ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 570c67a8140..478985923cb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -86,7 +86,6 @@ jobs:
           prefix-key: "0" # change this to invalidate CI cache
           shared-key: "cargo_nextest-${{ matrix.cache_id }}"
       - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
-      - run: echo "LLVM_PROFILE_FILE=$(pwd)/target/debug/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
 
       # Run unit tests
       - run: cargo nextest run --locked --workspace --exclude integration-tests --cargo-profile quick-release --profile ci ${{ matrix.flags }}

From b089cdf836a73109120148796d041470a83d1899 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Thu, 16 Nov 2023 01:37:31 +0000
Subject: [PATCH 10/24] let cargo llvm-cov find the right cargo profile

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 478985923cb..fbce5bf27a0 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -93,7 +93,7 @@ jobs:
           RUST_BACKTRACE: short
       - run: find . -name '*.profraw'
       - run: find target
-      - run: cargo llvm-cov report --codecov --output-path unittests.json
+      - run: cargo llvm-cov report --profile quick-release --codecov --output-path unittests.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:
           files: unittests.json

From b7571f99935ce6a320524a1eba3b5ddf6b374827 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Thu, 16 Nov 2023 01:59:05 +0000
Subject: [PATCH 11/24] try using what seems to work everywhere

---
 .github/workflows/ci.yml | 32 ++++++++++++--------------------
 1 file changed, 12 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index fbce5bf27a0..28a42953541 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -27,8 +27,7 @@ jobs:
           save-if: "false" # use the cache from nextest, but don’t double-save
       # TODO: remove the LLVM_PROFILE_FILE override once
       # https://github.com/taiki-e/cargo-llvm-cov/issues/320 is fixed (and same below)
-      - run: cargo llvm-cov show-env >> "$GITHUB_ENV"
-      - run: echo "LLVM_PROFILE_FILE=$(pwd)/target/llvm-cov-target/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
+      - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
       - run: cargo build --locked --profile quick-release -p neard --bin neard
       - uses: actions/upload-artifact@v3
         with:
@@ -91,8 +90,6 @@ jobs:
       - run: cargo nextest run --locked --workspace --exclude integration-tests --cargo-profile quick-release --profile ci ${{ matrix.flags }}
         env:
           RUST_BACKTRACE: short
-      - run: find . -name '*.profraw'
-      - run: find target
       - run: cargo llvm-cov report --profile quick-release --codecov --output-path unittests.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:
@@ -106,7 +103,7 @@ jobs:
       - run: cargo nextest run --locked --package integration-tests --cargo-profile quick-release --profile ci ${{ matrix.flags }}
         env:
           RUST_BACKTRACE: short
-      - run: cargo llvm-cov report --codecov --output-path integration-tests.json
+      - run: cargo llvm-cov report --profile quick-release --codecov --output-path integration-tests.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:
           files: integration-tests.json
@@ -146,10 +143,9 @@ jobs:
       - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r requirements.txt
-      - run: cargo llvm-cov show-env >> "$GITHUB_ENV"
-      - run: echo "LLVM_PROFILE_FILE=$(pwd)/../target/llvm-cov-target/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
+      - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
       - run: python3 tests/sanity/backward_compatible.py
-      - run: cargo llvm-cov report --codecov --output-path pytest-backcomp.json
+      - run: cargo llvm-cov report --profile quick-release --codecov --output-path pytest-backcomp.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:
           files: pytest-backcomp.json
@@ -180,10 +176,9 @@ jobs:
       - run: echo "NEAR_ROOT=$PWD" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r requirements.txt
-      - run: cargo llvm-cov show-env >> "$GITHUB_ENV"
-      - run: echo "LLVM_PROFILE_FILE=$(pwd)/../target/llvm-cov-target/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
+      - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
       - run: python3 tests/sanity/db_migration.py
-      - run: cargo llvm-cov report --codecov --output-path pytest-dbmigr.json
+      - run: cargo llvm-cov report --profile quick-release --codecov --output-path pytest-dbmigr.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:
           files: pytest-dbmigr.json
@@ -213,8 +208,7 @@ jobs:
       - run: pip3 install --user -r pytest/requirements.txt
       # This is the only job that uses `--features nightly` so we build this in-line instead of a
       # separate job like done with the regular neard.
-      - run: cargo llvm-cov show-env >> "$GITHUB_ENV"
-      - run: echo "LLVM_PROFILE_FILE=$(pwd)/target/llvm-cov-target/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
+      - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
       - run: cargo build --profile quick-release -p neard --bin neard --features nightly
       # Note: We're not running spin_up_cluster.py for non-nightly
       # because spinning up non-nightly clusters is already covered
@@ -222,7 +216,7 @@ jobs:
       - run: python3 pytest/tests/sanity/spin_up_cluster.py
         env:
           NEAR_ROOT: "target/quick-release"
-      - run: cargo llvm-cov report --codecov --output-path pytest-sanity.json
+      - run: cargo llvm-cov report --profile quick-release --codecov --output-path pytest-sanity.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:
           files: pytest-sanity.json
@@ -249,10 +243,9 @@ jobs:
       - run: echo "CURRENT_NEARD=$PWD/target/quick-release/neard" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r pytest/requirements.txt
-      - run: cargo llvm-cov show-env >> "$GITHUB_ENV"
-      - run: echo "LLVM_PROFILE_FILE=$(pwd)/target/llvm-cov-target/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
+      - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
       - run: python3 scripts/state/update_res.py check
-      - run: cargo llvm-cov report --codecov --output-path pytest-genesischk.json
+      - run: cargo llvm-cov report --profile quick-release --codecov --output-path pytest-genesischk.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:
           files: pytest-genesischk.json
@@ -297,10 +290,9 @@ jobs:
       - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r requirements.txt
-      - run: cargo llvm-cov show-env >> "$GITHUB_ENV"
-      - run: echo "LLVM_PROFILE_FILE=$(pwd)/target/llvm-cov-target/nearcore-%p-%16m.profraw" >> "$GITHUB_ENV"
+      - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
       - run: python3 tests/sanity/upgradable.py
-      - run: cargo llvm-cov report --codecov --output-path pytest-upgradability.json
+      - run: cargo llvm-cov report --profile quick-release --codecov --output-path pytest-upgradability.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:
           files: pytest-upgradability.json

From 41d72b81c201121f13ca6bc5d356fdc3f58b915a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Thu, 16 Nov 2023 02:14:04 +0000
Subject: [PATCH 12/24] =?UTF-8?q?debug=20what=E2=80=99s=20going=20wrong=20?=
 =?UTF-8?q?with=20pytests?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .github/workflows/ci.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 28a42953541..7fa1dc36a42 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -145,6 +145,9 @@ jobs:
       - run: pip3 install --user -r requirements.txt
       - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
       - run: python3 tests/sanity/backward_compatible.py
+      - run: find .. -name '*.profraw'
+      - run: find ../target
+      - run: find ..
       - run: cargo llvm-cov report --profile quick-release --codecov --output-path pytest-backcomp.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:

From dc548a10a50ec5eb356fd5d1279809e42047e0b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Thu, 16 Nov 2023 02:17:30 +0000
Subject: [PATCH 13/24] actually delete all profraw files

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7fa1dc36a42..2600421275d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -97,7 +97,7 @@ jobs:
           fail_ci_if_error: true
           flags: unittests,${{ matrix.id }}
       # See https://github.com/taiki-e/cargo-llvm-cov/issues/292
-      - run: rm -f target/llvm-cov-target/*.profraw
+      - run: find target -name '*.profraw' -delete
 
       # Run integration tests
       - run: cargo nextest run --locked --package integration-tests --cargo-profile quick-release --profile ci ${{ matrix.flags }}

From e7c20cd5d8ffbe400e6638713c5312ca72142f54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Thu, 16 Nov 2023 02:22:30 +0000
Subject: [PATCH 14/24] disable integration-tests for macos at the right place

---
 .github/workflows/ci.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2600421275d..11831c1f731 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -59,7 +59,8 @@ jobs:
             os: macos-latest-xlarge
             # FIXME: some of these tests don't work very well on MacOS at the moment. Should fix
             # them at earliest convenience :)
-            flags: "--exclude integration-tests --exclude node-runtime --exclude runtime-params-estimator --exclude near-network --exclude estimator-warehouse"
+            # Note that integration-tests are disabled at the step running them in the job below.
+            flags: "--exclude node-runtime --exclude runtime-params-estimator --exclude near-network --exclude estimator-warehouse"
     timeout-minutes: 90
     steps:
       # Some of the tests allocate really sparse maps, so heuristic-based overcommit limits are not
@@ -101,10 +102,13 @@ jobs:
 
       # Run integration tests
       - run: cargo nextest run --locked --package integration-tests --cargo-profile quick-release --profile ci ${{ matrix.flags }}
+        if: matrix.id != 'macos'
         env:
           RUST_BACKTRACE: short
       - run: cargo llvm-cov report --profile quick-release --codecov --output-path integration-tests.json
+        if: matrix.id != 'macos'
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
+        if: matrix.id != 'macos'
         with:
           files: integration-tests.json
           fail_ci_if_error: true

From 557baedb10d8256974dcd894b404a4e067770127 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Thu, 16 Nov 2023 11:21:49 +0000
Subject: [PATCH 15/24] try creating the target folder early?

---
 .github/workflows/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 11831c1f731..c219b7be35b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -147,6 +147,7 @@ jobs:
       - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r requirements.txt
+      - run: mkdir ../target
       - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
       - run: python3 tests/sanity/backward_compatible.py
       - run: find .. -name '*.profraw'

From d4c5a799ec003aef1528ee4cd0c085dea601baae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Thu, 16 Nov 2023 11:33:24 +0000
Subject: [PATCH 16/24] try not using pytest as workdir

---
 .github/workflows/ci.yml | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c219b7be35b..57a611fc528 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -128,9 +128,6 @@ jobs:
     name: "Backward Compatibility"
     needs: build_binary
     runs-on: ubuntu-22.04
-    defaults:
-      run:
-        working-directory: ./pytest
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
@@ -144,15 +141,14 @@ jobs:
         with:
           name: neard
           path: pytest # NB: this does not account for defaults.run.working-directory
-      - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV"
+      - run: echo "CURRENT_NEARD=$PWD/pytest/neard" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
-      - run: pip3 install --user -r requirements.txt
-      - run: mkdir ../target
+      - run: pip3 install --user -r pytest/requirements.txt
       - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
-      - run: python3 tests/sanity/backward_compatible.py
-      - run: find .. -name '*.profraw'
-      - run: find ../target
-      - run: find ..
+      - run: cd pytest && python3 tests/sanity/backward_compatible.py
+      - run: find . -name '*.profraw'
+      - run: find target
+      - run: find .
       - run: cargo llvm-cov report --profile quick-release --codecov --output-path pytest-backcomp.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:

From 41d555cca5674a29581f17887daa60363667688d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Thu, 16 Nov 2023 12:30:14 +0000
Subject: [PATCH 17/24] and with creating target folder?

---
 .github/workflows/ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 57a611fc528..e6fd46157cf 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -145,6 +145,8 @@ jobs:
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r pytest/requirements.txt
       - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
+      - run: find .
+      - run: mkdir target
       - run: cd pytest && python3 tests/sanity/backward_compatible.py
       - run: find . -name '*.profraw'
       - run: find target

From 4ad3102bc2c3e8fb8bf922f441990ac3ad8cea5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Thu, 16 Nov 2023 12:38:09 +0000
Subject: [PATCH 18/24] try uploading the whole target folder

---
 .github/workflows/ci.yml | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e6fd46157cf..32f98c6df97 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -31,8 +31,8 @@ jobs:
       - run: cargo build --locked --profile quick-release -p neard --bin neard
       - uses: actions/upload-artifact@v3
         with:
-          name: neard
-          path: target/quick-release/neard
+          name: neard_target
+          path: target
           if-no-files-found: error
           retention-days: 1
 
@@ -139,9 +139,9 @@ jobs:
           crate: cargo-llvm-cov
       - uses: actions/download-artifact@v3
         with:
-          name: neard
-          path: pytest # NB: this does not account for defaults.run.working-directory
-      - run: echo "CURRENT_NEARD=$PWD/pytest/neard" >> "$GITHUB_ENV"
+          name: neard_target
+          path: . # NB: this does not account for defaults.run.working-directory
+      - run: echo "CURRENT_NEARD=$PWD/../target/quick-release/neard" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r pytest/requirements.txt
       - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
@@ -176,9 +176,9 @@ jobs:
           crate: cargo-llvm-cov
       - uses: actions/download-artifact@v3
         with:
-          name: neard
-          path: pytest # NB: this does not account for defaults.run.working-directory
-      - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV"
+          name: neard_target
+          path: . # NB: this does not account for defaults.run.working-directory
+      - run: echo "CURRENT_NEARD=$PWD/../target/quick-release/neard" >> "$GITHUB_ENV"
       - run: echo "NEAR_ROOT=$PWD" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r requirements.txt
@@ -244,8 +244,8 @@ jobs:
           crate: cargo-llvm-cov
       - uses: actions/download-artifact@v3
         with:
-          name: neard
-          path: target/quick-release
+          name: neard_target
+          path: .
       - run: echo "CURRENT_NEARD=$PWD/target/quick-release/neard" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r pytest/requirements.txt
@@ -291,9 +291,9 @@ jobs:
           crate: cargo-llvm-cov
       - uses: actions/download-artifact@v3
         with:
-          name: neard
-          path: pytest # NB: this does not account for defaults.run.working-directory
-      - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV"
+          name: neard_target
+          path: . # NB: this does not account for defaults.run.working-directory
+      - run: echo "CURRENT_NEARD=$PWD/../target/quick-release/neard" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r requirements.txt
       - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"

From a9748876cfdaa7bcfd61bd3dd484fb1b69a37cbf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Thu, 16 Nov 2023 12:49:37 +0000
Subject: [PATCH 19/24] compress target before uploading

---
 .github/workflows/ci.yml | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 32f98c6df97..33e98097500 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -29,10 +29,11 @@ jobs:
       # https://github.com/taiki-e/cargo-llvm-cov/issues/320 is fixed (and same below)
       - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
       - run: cargo build --locked --profile quick-release -p neard --bin neard
+      - run: tar czf target.tar.gz target
       - uses: actions/upload-artifact@v3
         with:
-          name: neard_target
-          path: target
+          name: target.tar.gz
+          path: target.tar.gz
           if-no-files-found: error
           retention-days: 1
 
@@ -139,9 +140,10 @@ jobs:
           crate: cargo-llvm-cov
       - uses: actions/download-artifact@v3
         with:
-          name: neard_target
+          name: target.tar.gz
           path: . # NB: this does not account for defaults.run.working-directory
-      - run: echo "CURRENT_NEARD=$PWD/../target/quick-release/neard" >> "$GITHUB_ENV"
+      - run: tar xzf target.tar.gz
+      - run: echo "CURRENT_NEARD=$PWD/target/quick-release/neard" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r pytest/requirements.txt
       - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
@@ -176,8 +178,9 @@ jobs:
           crate: cargo-llvm-cov
       - uses: actions/download-artifact@v3
         with:
-          name: neard_target
+          name: target.tar.gz
           path: . # NB: this does not account for defaults.run.working-directory
+      - run: cd .. && tar xzf target.tar.gz
       - run: echo "CURRENT_NEARD=$PWD/../target/quick-release/neard" >> "$GITHUB_ENV"
       - run: echo "NEAR_ROOT=$PWD" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
@@ -244,8 +247,9 @@ jobs:
           crate: cargo-llvm-cov
       - uses: actions/download-artifact@v3
         with:
-          name: neard_target
+          name: target.tar.gz
           path: .
+      - run: tar xzf target.tar.gz
       - run: echo "CURRENT_NEARD=$PWD/target/quick-release/neard" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r pytest/requirements.txt
@@ -291,8 +295,9 @@ jobs:
           crate: cargo-llvm-cov
       - uses: actions/download-artifact@v3
         with:
-          name: neard_target
+          name: target.tar.gz
           path: . # NB: this does not account for defaults.run.working-directory
+      - run: cd .. && tar xzf target.tar.gz
       - run: echo "CURRENT_NEARD=$PWD/../target/quick-release/neard" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r requirements.txt

From 6705812f6f03985d2821dabcb74f5a655b3c0a4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Thu, 16 Nov 2023 13:17:45 +0000
Subject: [PATCH 20/24] do not upload whole target folder

---
 .github/workflows/ci.yml | 31 +++++++++++++------------------
 1 file changed, 13 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 33e98097500..e6fd46157cf 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -29,11 +29,10 @@ jobs:
       # https://github.com/taiki-e/cargo-llvm-cov/issues/320 is fixed (and same below)
       - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
       - run: cargo build --locked --profile quick-release -p neard --bin neard
-      - run: tar czf target.tar.gz target
       - uses: actions/upload-artifact@v3
         with:
-          name: target.tar.gz
-          path: target.tar.gz
+          name: neard
+          path: target/quick-release/neard
           if-no-files-found: error
           retention-days: 1
 
@@ -140,10 +139,9 @@ jobs:
           crate: cargo-llvm-cov
       - uses: actions/download-artifact@v3
         with:
-          name: target.tar.gz
-          path: . # NB: this does not account for defaults.run.working-directory
-      - run: tar xzf target.tar.gz
-      - run: echo "CURRENT_NEARD=$PWD/target/quick-release/neard" >> "$GITHUB_ENV"
+          name: neard
+          path: pytest # NB: this does not account for defaults.run.working-directory
+      - run: echo "CURRENT_NEARD=$PWD/pytest/neard" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r pytest/requirements.txt
       - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
@@ -178,10 +176,9 @@ jobs:
           crate: cargo-llvm-cov
       - uses: actions/download-artifact@v3
         with:
-          name: target.tar.gz
-          path: . # NB: this does not account for defaults.run.working-directory
-      - run: cd .. && tar xzf target.tar.gz
-      - run: echo "CURRENT_NEARD=$PWD/../target/quick-release/neard" >> "$GITHUB_ENV"
+          name: neard
+          path: pytest # NB: this does not account for defaults.run.working-directory
+      - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV"
       - run: echo "NEAR_ROOT=$PWD" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r requirements.txt
@@ -247,9 +244,8 @@ jobs:
           crate: cargo-llvm-cov
       - uses: actions/download-artifact@v3
         with:
-          name: target.tar.gz
-          path: .
-      - run: tar xzf target.tar.gz
+          name: neard
+          path: target/quick-release
       - run: echo "CURRENT_NEARD=$PWD/target/quick-release/neard" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r pytest/requirements.txt
@@ -295,10 +291,9 @@ jobs:
           crate: cargo-llvm-cov
       - uses: actions/download-artifact@v3
         with:
-          name: target.tar.gz
-          path: . # NB: this does not account for defaults.run.working-directory
-      - run: cd .. && tar xzf target.tar.gz
-      - run: echo "CURRENT_NEARD=$PWD/../target/quick-release/neard" >> "$GITHUB_ENV"
+          name: neard
+          path: pytest # NB: this does not account for defaults.run.working-directory
+      - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV"
       - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r requirements.txt
       - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"

From ae63426673141a418463be355454c9323f1d3f7f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Thu, 16 Nov 2023 13:27:51 +0000
Subject: [PATCH 21/24] discard preparatory build neard job

---
 .github/workflows/ci.yml | 97 ++++++++++++++--------------------------
 1 file changed, 33 insertions(+), 64 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e6fd46157cf..27ee0ed2ef0 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -12,30 +12,6 @@ env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  build_binary:
-    name: "Build neard"
-    runs-on: ubuntu-22.04-16core
-    steps:
-      - uses: actions/checkout@v4
-      - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
-        with:
-          crate: cargo-llvm-cov
-      - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
-        with:
-          prefix-key: "0" # change this to invalidate CI cache
-          shared-key: "cargo_nextest-linux"
-          save-if: "false" # use the cache from nextest, but don’t double-save
-      # TODO: remove the LLVM_PROFILE_FILE override once
-      # https://github.com/taiki-e/cargo-llvm-cov/issues/320 is fixed (and same below)
-      - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
-      - run: cargo build --locked --profile quick-release -p neard --bin neard
-      - uses: actions/upload-artifact@v3
-        with:
-          name: neard
-          path: target/quick-release/neard
-          if-no-files-found: error
-          retention-days: 1
-
   cargo_nextest:
     name: "Cargo Nextest (${{matrix.name}})"
     runs-on: ${{ matrix.os }}
@@ -116,7 +92,7 @@ jobs:
 
   protobuf_backward_compat:
     name: "Protobuf Backward Compatibility"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04-8core
     steps:
       - uses: actions/checkout@v4
       - uses: bufbuild/buf-setup-action@1158f4fa81bc02e1ff62abcca6d516c9e24c77da
@@ -126,7 +102,6 @@ jobs:
 
   py_backward_compat:
     name: "Backward Compatibility"
-    needs: build_binary
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
@@ -137,16 +112,16 @@ jobs:
       - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
         with:
           crate: cargo-llvm-cov
-      - uses: actions/download-artifact@v3
+      - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
         with:
-          name: neard
-          path: pytest # NB: this does not account for defaults.run.working-directory
-      - run: echo "CURRENT_NEARD=$PWD/pytest/neard" >> "$GITHUB_ENV"
-      - run: chmod +x "$CURRENT_NEARD"
+          prefix-key: "0" # change this to invalidate CI cache
+          shared-key: "cargo_nextest-linux"
+          save-if: "false" # use the cache from nextest, but don’t double-save
       - run: pip3 install --user -r pytest/requirements.txt
       - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
+      - run: cargo build --locked --profile quick-release -p neard --bin neard
+      - run: echo "CURRENT_NEARD=$PWD/target/quick-release/neard" >> "$GITHUB_ENV"
       - run: find .
-      - run: mkdir target
       - run: cd pytest && python3 tests/sanity/backward_compatible.py
       - run: find . -name '*.profraw'
       - run: find target
@@ -160,11 +135,7 @@ jobs:
 
   py_db_migration:
     name: "Database Migration"
-    needs: build_binary
-    runs-on: ubuntu-22.04
-    defaults:
-      run:
-        working-directory: ./pytest
+    runs-on: ubuntu-22.04-8core
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
@@ -174,16 +145,17 @@ jobs:
       - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
         with:
           crate: cargo-llvm-cov
-      - uses: actions/download-artifact@v3
+      - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
         with:
-          name: neard
-          path: pytest # NB: this does not account for defaults.run.working-directory
-      - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV"
-      - run: echo "NEAR_ROOT=$PWD" >> "$GITHUB_ENV"
-      - run: chmod +x "$CURRENT_NEARD"
-      - run: pip3 install --user -r requirements.txt
+          prefix-key: "0" # change this to invalidate CI cache
+          shared-key: "cargo_nextest-linux"
+          save-if: "false" # use the cache from nextest, but don’t double-save
+      - run: pip3 install --user -r pytest/requirements.txt
       - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
-      - run: python3 tests/sanity/db_migration.py
+      - run: cargo build --locked --profile quick-release -p neard --bin neard
+      - run: echo "CURRENT_NEARD=$PWD/target/quick-release/neard" >> "$GITHUB_ENV"
+      - run: echo "NEAR_ROOT=$PWD" >> "$GITHUB_ENV"
+      - run: cd pytest && python3 tests/sanity/db_migration.py
       - run: cargo llvm-cov report --profile quick-release --codecov --output-path pytest-dbmigr.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:
@@ -231,8 +203,7 @@ jobs:
 
   py_genesis_check:
     name: "Genesis Changes"
-    needs: build_binary
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-22.04-8core
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
@@ -242,14 +213,15 @@ jobs:
       - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
         with:
           crate: cargo-llvm-cov
-      - uses: actions/download-artifact@v3
+      - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
         with:
-          name: neard
-          path: target/quick-release
-      - run: echo "CURRENT_NEARD=$PWD/target/quick-release/neard" >> "$GITHUB_ENV"
-      - run: chmod +x "$CURRENT_NEARD"
+          prefix-key: "0" # change this to invalidate CI cache
+          shared-key: "cargo_nextest-linux"
+          save-if: "false" # use the cache from nextest, but don’t double-save
       - run: pip3 install --user -r pytest/requirements.txt
       - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
+      - run: cargo build --locked --profile quick-release -p neard --bin neard
+      - run: echo "CURRENT_NEARD=$PWD/target/quick-release/neard" >> "$GITHUB_ENV"
       - run: python3 scripts/state/update_res.py check
       - run: cargo llvm-cov report --profile quick-release --codecov --output-path pytest-genesischk.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
@@ -275,11 +247,7 @@ jobs:
 
   py_upgradability:
     name: "Upgradability"
-    needs: build_binary
-    runs-on: ubuntu-22.04
-    defaults:
-      run:
-        working-directory: ./pytest
+    runs-on: ubuntu-22.04-8core
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
@@ -289,15 +257,16 @@ jobs:
       - uses: baptiste0928/cargo-install@21a18ba3bf4a184d1804e8b759930d3471b1c941
         with:
           crate: cargo-llvm-cov
-      - uses: actions/download-artifact@v3
+      - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
         with:
-          name: neard
-          path: pytest # NB: this does not account for defaults.run.working-directory
-      - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV"
-      - run: chmod +x "$CURRENT_NEARD"
-      - run: pip3 install --user -r requirements.txt
+          prefix-key: "0" # change this to invalidate CI cache
+          shared-key: "cargo_nextest-linux"
+          save-if: "false" # use the cache from nextest, but don’t double-save
+      - run: pip3 install --user -r pytest/requirements.txt
       - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
-      - run: python3 tests/sanity/upgradable.py
+      - run: cargo build --locked --profile quick-release -p neard --bin neard
+      - run: echo "CURRENT_NEARD=$PWD/target/quick-release/neard" >> "$GITHUB_ENV"
+      - run: cd pytest && python3 tests/sanity/upgradable.py
       - run: cargo llvm-cov report --profile quick-release --codecov --output-path pytest-upgradability.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:

From edccf7cdc7830031d445d647b2a25f0098375398 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Thu, 16 Nov 2023 13:50:57 +0000
Subject: [PATCH 22/24] cleanup test lines

---
 .github/workflows/ci.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 27ee0ed2ef0..d5e1096f11c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -121,11 +121,7 @@ jobs:
       - run: cargo llvm-cov show-env | tr -d "'" >> "$GITHUB_ENV"
       - run: cargo build --locked --profile quick-release -p neard --bin neard
       - run: echo "CURRENT_NEARD=$PWD/target/quick-release/neard" >> "$GITHUB_ENV"
-      - run: find .
       - run: cd pytest && python3 tests/sanity/backward_compatible.py
-      - run: find . -name '*.profraw'
-      - run: find target
-      - run: find .
       - run: cargo llvm-cov report --profile quick-release --codecov --output-path pytest-backcomp.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:

From f1fcc7ca3bd0e7825391069d1be950c800aa20fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Fri, 17 Nov 2023 11:06:01 +0000
Subject: [PATCH 23/24] review comments

---
 .github/workflows/ci.yml | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d5e1096f11c..9251e4f0f0d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -8,9 +8,6 @@ on:
   pull_request:
   merge_group:
 
-env:
-  CARGO_TERM_COLOR: always
-
 jobs:
   cargo_nextest:
     name: "Cargo Nextest (${{matrix.name}})"
@@ -24,19 +21,21 @@ jobs:
             cache_id: linux
             os: ubuntu-22.04-16core
             flags: ""
+            run_integ_tests: true
           - name: Linux Nightly
             id: linux-nightly
             cache_id: linux
             os: ubuntu-22.04-16core
             flags: "--features nightly,test_features"
+            run_integ_tests: true
           - name: MacOS
             id: macos
             cache_id: macos
             os: macos-latest-xlarge
             # FIXME: some of these tests don't work very well on MacOS at the moment. Should fix
             # them at earliest convenience :)
-            # Note that integration-tests are disabled at the step running them in the job below.
             flags: "--exclude node-runtime --exclude runtime-params-estimator --exclude near-network --exclude estimator-warehouse"
+            run_integ_tests: false
     timeout-minutes: 90
     steps:
       # Some of the tests allocate really sparse maps, so heuristic-based overcommit limits are not
@@ -78,13 +77,13 @@ jobs:
 
       # Run integration tests
       - run: cargo nextest run --locked --package integration-tests --cargo-profile quick-release --profile ci ${{ matrix.flags }}
-        if: matrix.id != 'macos'
+        if: matrix.run_integ_tests
         env:
           RUST_BACKTRACE: short
       - run: cargo llvm-cov report --profile quick-release --codecov --output-path integration-tests.json
-        if: matrix.id != 'macos'
+        if: matrix.run_integ_tests
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
-        if: matrix.id != 'macos'
+        if: matrix.run_integ_tests
         with:
           files: integration-tests.json
           fail_ci_if_error: true

From a1c19766de02b17d050d8c3937374cd378168ccf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Gaspard?= <leo@near.org>
Date: Mon, 20 Nov 2023 09:25:36 +0000
Subject: [PATCH 24/24] upload with codecov token to avoid intermittent failure
 to autodetect the repo

---
 .github/workflows/ci.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9251e4f0f0d..1f43e0919ab 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -69,6 +69,7 @@ jobs:
       - run: cargo llvm-cov report --profile quick-release --codecov --output-path unittests.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:
+          token: ${{ secrets.CODECOV_TOKEN }}
           files: unittests.json
           fail_ci_if_error: true
           flags: unittests,${{ matrix.id }}
@@ -85,6 +86,7 @@ jobs:
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         if: matrix.run_integ_tests
         with:
+          token: ${{ secrets.CODECOV_TOKEN }}
           files: integration-tests.json
           fail_ci_if_error: true
           flags: integration-tests,${{ matrix.id }}
@@ -124,6 +126,7 @@ jobs:
       - run: cargo llvm-cov report --profile quick-release --codecov --output-path pytest-backcomp.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:
+          token: ${{ secrets.CODECOV_TOKEN }}
           files: pytest-backcomp.json
           fail_ci_if_error: true
           flags: pytests,backward-compatibility,linux
@@ -154,6 +157,7 @@ jobs:
       - run: cargo llvm-cov report --profile quick-release --codecov --output-path pytest-dbmigr.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:
+          token: ${{ secrets.CODECOV_TOKEN }}
           files: pytest-dbmigr.json
           fail_ci_if_error: true
           flags: pytests,db-migration,linux
@@ -192,6 +196,7 @@ jobs:
       - run: cargo llvm-cov report --profile quick-release --codecov --output-path pytest-sanity.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:
+          token: ${{ secrets.CODECOV_TOKEN }}
           files: pytest-sanity.json
           fail_ci_if_error: true
           flags: pytests,sanity-checks,linux-nightly
@@ -221,6 +226,7 @@ jobs:
       - run: cargo llvm-cov report --profile quick-release --codecov --output-path pytest-genesischk.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:
+          token: ${{ secrets.CODECOV_TOKEN }}
           files: pytest-genesischk.json
           fail_ci_if_error: true
           flags: pytests,genesis-check,linux
@@ -265,6 +271,7 @@ jobs:
       - run: cargo llvm-cov report --profile quick-release --codecov --output-path pytest-upgradability.json
       - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
         with:
+          token: ${{ secrets.CODECOV_TOKEN }}
           files: pytest-upgradability.json
           fail_ci_if_error: true
           flags: pytests,upgradability,linux