From 15cda8a777644a1f612f9f2ca86a39f8ff98b02e Mon Sep 17 00:00:00 2001 From: hardfist Date: Mon, 18 May 2026 15:56:00 +0800 Subject: [PATCH 1/6] feat: switch continuous benchmarks to gungraun --- .github/workflows/benchmark.yml | 38 +++++---- Cargo.lock | 99 ++++++++++++++++++++++++ Cargo.toml | 5 ++ benches/resolver_gungraun.rs | 18 +++++ src/bin/resolver_bench_driver.rs | 127 +++++++++++++++++++++++++++++++ 5 files changed, 267 insertions(+), 20 deletions(-) create mode 100644 benches/resolver_gungraun.rs create mode 100644 src/bin/resolver_bench_driver.rs diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 9a1bd5d9..3a2fa00e 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -21,7 +21,7 @@ permissions: jobs: benchmark: - name: Benchmark + name: Benchmark (Gungraun) runs-on: ubuntu-latest steps: - name: Checkout Branch @@ -41,31 +41,29 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 with: - cache-key: benchmark + cache-key: benchmark-gungraun cache-save-if: ${{ github.ref_name == 'main' }} + - uses: ./.github/actions/pnpm + + - name: Install valgrind + run: sudo apt-get update && sudo apt-get install -y valgrind + - uses: taiki-e/install-action@055f5df8c3f65ea01cd41e9dc855becd88953486 # v2.75.18 with: - tool: cargo-codspeed + tool: gungraun-runner - - uses: ./.github/actions/pnpm - - name: Build Benchmark + - name: Run gungraun benchmark + timeout-minutes: 45 env: + GUNGRAUN_VALGRIND_BIN: /usr/bin/valgrind RUSTFLAGS: "-C debuginfo=1 -C strip=none -g" - run: cargo codspeed build - - - name: Run CPU benchmark - uses: CodSpeedHQ/action@281164b0f014a4e7badd2c02cecad9b595b70537 # https://github.com/CodSpeedHQ/action/releases/tag/v4.10.6 - timeout-minutes: 30 - with: - mode: simulation - run: cargo codspeed run - token: ${{ secrets.CODSPEED_TOKEN }} + run: cargo bench --bench resolver_gungraun -- --nocapture - - name: Run memory benchmark - uses: CodSpeedHQ/action@281164b0f014a4e7badd2c02cecad9b595b70537 # https://github.com/CodSpeedHQ/action/releases/tag/v4.10.6 - timeout-minutes: 30 + - name: Upload gungraun result files + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - mode: memory - run: cargo codspeed run - token: ${{ secrets.CODSPEED_TOKEN }} + name: gungraun-target + path: target/gungraun + if-no-files-found: ignore diff --git a/Cargo.lock b/Cargo.lock index 0e3ed41f..98bbb677 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,6 +76,15 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -341,6 +350,27 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn", +] + [[package]] name = "document-features" version = "0.2.12" @@ -534,6 +564,43 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "gungraun" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bfa183cef2c88324b20de9727befd262c35592d9885ffd93b8b102536a81cf2" +dependencies = [ + "bincode", + "derive_more", + "gungraun-macros", + "gungraun-runner", +] + +[[package]] +name = "gungraun-macros" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e35c7fb6133421db1cf752b7a2838d9277a26810ccaeeca7aa449f96ad7c2b01" +dependencies = [ + "derive_more", + "proc-macro-error2", + "proc-macro2", + "quote", + "rustc_version", + "serde", + "serde_json", + "syn", +] + +[[package]] +name = "gungraun-runner" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ebdac6d5fe0aa9d9623d690fbfdb759e0ed44115e0685b9abc4568fe47db364" +dependencies = [ + "serde", +] + [[package]] name = "half" version = "2.7.1" @@ -897,6 +964,28 @@ dependencies = [ "thiserror", ] +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.103" @@ -1027,6 +1116,7 @@ dependencies = [ "document-features", "dunce", "futures", + "gungraun", "indexmap", "json-strip-comments", "mimalloc", @@ -1050,6 +1140,15 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustversion" version = "1.0.22" diff --git a/Cargo.toml b/Cargo.toml index 6c362070..3bbf9e1c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,10 @@ doctest = false harness = false name = "resolver" +[[bench]] +harness = false +name = "resolver_gungraun" + [lints.clippy] all = { level = "warn", priority = -1 } cargo = { level = "warn", priority = -1 } @@ -107,6 +111,7 @@ tokio = { version = "1.48.0", default-features = false, features = ["sync", "rt" criterion = { version = "4.3.0", package = "codspeed-criterion-compat", default-features = false, features = [ "async_tokio", ] } +gungraun = "0.18.2" normalize-path = { version = "0.2.1" } rayon = { version = "1.11.0" } diff --git a/benches/resolver_gungraun.rs b/benches/resolver_gungraun.rs new file mode 100644 index 00000000..9750ae7b --- /dev/null +++ b/benches/resolver_gungraun.rs @@ -0,0 +1,18 @@ +use gungraun::{binary_benchmark, binary_benchmark_group, main, BinaryBenchmarkConfig, Command}; + +#[binary_benchmark] +#[bench::resolve_dependencies("deps")] +#[bench::resolve_many_extensions("many_extensions")] +#[bench::resolve_dependencies_pnp("pnp")] +fn bench_resolver(scenario: &str) -> Command { + Command::new(env!("CARGO_BIN_EXE_resolver_bench_driver")) + .arg(scenario) + .build() +} + +binary_benchmark_group!(name = resolver_group, benchmarks = bench_resolver); + +main!( + config = BinaryBenchmarkConfig::default().env_clear(false), + binary_benchmark_groups = resolver_group +); diff --git a/src/bin/resolver_bench_driver.rs b/src/bin/resolver_bench_driver.rs new file mode 100644 index 00000000..5bc7b785 --- /dev/null +++ b/src/bin/resolver_bench_driver.rs @@ -0,0 +1,127 @@ +use std::{env, fs::read_to_string, path::PathBuf, process}; + +use rspack_resolver::{AliasValue, FileSystemOptions, FileSystemOs, ResolveOptions, Resolver}; +use serde_json::Value; + +fn rspack_resolver(enable_pnp: bool) -> Resolver { + #[cfg(not(feature = "yarn_pnp"))] + let _ = enable_pnp; + + let alias_value = AliasValue::from("./"); + let fs = FileSystemOs::new(FileSystemOptions { + #[cfg(feature = "yarn_pnp")] + enable_pnp, + }); + + Resolver::new_with_file_system( + fs, + ResolveOptions { + #[cfg(feature = "yarn_pnp")] + enable_pnp, + extensions: vec![".ts".into(), ".js".into(), ".mjs".into()], + condition_names: vec!["import".into(), "webpack".into(), "require".into()], + alias_fields: vec![vec!["browser".into()]], + extension_alias: vec![(".js".into(), vec![".ts".into(), ".js".into()])], + alias: vec![ + ("/absolute/path".into(), vec![alias_value.clone()]), + ("aaa".into(), vec![alias_value.clone()]), + ("bbb".into(), vec![alias_value.clone()]), + ("ccc".into(), vec![alias_value.clone()]), + ("ddd".into(), vec![alias_value.clone()]), + ("eee".into(), vec![alias_value.clone()]), + ("fff".into(), vec![alias_value.clone()]), + ("ggg".into(), vec![alias_value.clone()]), + ("hhh".into(), vec![alias_value.clone()]), + ("iii".into(), vec![alias_value.clone()]), + ("jjj".into(), vec![alias_value.clone()]), + ("kkk".into(), vec![alias_value.clone()]), + ("lll".into(), vec![alias_value.clone()]), + ("mmm".into(), vec![alias_value.clone()]), + ("nnn".into(), vec![alias_value.clone()]), + ("ooo".into(), vec![alias_value.clone()]), + ("ppp".into(), vec![alias_value.clone()]), + ("qqq".into(), vec![alias_value.clone()]), + ("rrr".into(), vec![alias_value.clone()]), + ("sss".into(), vec![alias_value.clone()]), + ("@".into(), vec![alias_value.clone()]), + ("@@".into(), vec![alias_value.clone()]), + ("@@@".into(), vec![alias_value]), + ], + ..ResolveOptions::default() + }, + ) +} + +fn resolver_with_many_extensions() -> Resolver { + Resolver::new(ResolveOptions { + extensions: vec![ + ".bad0".to_string(), + ".bad1".to_string(), + ".bad2".to_string(), + ".bad3".to_string(), + ".bad4".to_string(), + ".bad5".to_string(), + ".bad6".to_string(), + ".bad7".to_string(), + ".bad8".to_string(), + ".bad9".to_string(), + ".mtsx".to_string(), + ".mts".to_string(), + ".mjs".to_string(), + ".tsx".to_string(), + ".ts".to_string(), + ".jsx".to_string(), + ".js".to_string(), + ], + imports_fields: vec![], + exports_fields: vec![], + enable_pnp: false, + ..ResolveOptions::default() + }) +} + +fn benchmark_requests() -> (PathBuf, Vec) { + let context = env::current_dir().unwrap().join("benches"); + let pkg_content = read_to_string("./benches/package.json").unwrap(); + let pkg_json: Value = serde_json::from_str(&pkg_content).unwrap(); + let requests = pkg_json["dependencies"] + .as_object() + .unwrap() + .keys() + .cloned() + .collect::>(); + (context, requests) +} + +async fn run_resolve_dependencies(enable_pnp: bool) { + let (context, requests) = benchmark_requests(); + let resolver = rspack_resolver(enable_pnp); + + for request in requests { + let _ = resolver.resolve(&context, &request).await; + } +} + +async fn run_resolve_many_extensions() { + let (context, requests) = benchmark_requests(); + let resolver = resolver_with_many_extensions(); + + for request in requests.iter().take(200) { + let _ = resolver.resolve(&context, request).await; + } +} + +#[tokio::main(flavor = "current_thread")] +async fn main() { + let scenario = env::args().nth(1).unwrap_or_else(|| "deps".to_string()); + + match scenario.as_str() { + "deps" => run_resolve_dependencies(false).await, + "many_extensions" => run_resolve_many_extensions().await, + "pnp" => run_resolve_dependencies(true).await, + _ => { + eprintln!("unknown scenario: {scenario}. expected one of: deps, many_extensions, pnp"); + process::exit(2); + } + } +} From 8cf14c2be851d86ef083ba0c7dc60599d17511b9 Mon Sep 17 00:00:00 2001 From: hardfist Date: Mon, 18 May 2026 15:59:01 +0800 Subject: [PATCH 2/6] ci: pin cargo-deny to 0.19.4 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ff9f959..51f69dcb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -120,7 +120,7 @@ jobs: cache: false - uses: taiki-e/install-action@055f5df8c3f65ea01cd41e9dc855becd88953486 # v2.75.18 with: - tool: cargo-deny + tool: cargo-deny@0.19.4 - if: steps.filter.outputs.src == 'true' run: cargo deny check From 6aacd59bfebdd193fa2dbfa67b6952f52ff1d28a Mon Sep 17 00:00:00 2001 From: hardfist Date: Mon, 18 May 2026 16:02:48 +0800 Subject: [PATCH 3/6] ci: ignore bincode advisory from gungraun transitive dep --- deny.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/deny.toml b/deny.toml index 1c4e9974..e0fd09c2 100644 --- a/deny.toml +++ b/deny.toml @@ -74,6 +74,7 @@ ignore = [ #{ id = "RUSTSEC-0000-0000", reason = "you can specify a reason the advisory is ignored" }, #"a-crate-that-is-yanked@0.1.1", # you can also ignore yanked crate versions if you wish #{ crate = "a-crate-that-is-yanked@0.1.1", reason = "you can specify why you are ignoring the yanked crate" }, + { id = "RUSTSEC-2025-0141", reason = "Transitive dependency from gungraun benchmark tooling; no safe upgrade path in current upstream release." }, ] # If this is true, then cargo deny will use the git executable to fetch advisory database. # If this is false, then it uses a built-in git library. From 78151f6f415122a6bdcd701538c6562ad01b7142 Mon Sep 17 00:00:00 2001 From: hardfist Date: Mon, 18 May 2026 16:06:38 +0800 Subject: [PATCH 4/6] ci: pin gungraun-runner to 0.18.2 --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 3a2fa00e..3eeba986 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -51,7 +51,7 @@ jobs: - uses: taiki-e/install-action@055f5df8c3f65ea01cd41e9dc855becd88953486 # v2.75.18 with: - tool: gungraun-runner + tool: gungraun-runner@0.18.2 - name: Run gungraun benchmark timeout-minutes: 45 From 598918bfb31ced5ce785d8e31e72b536c4fc2f20 Mon Sep 17 00:00:00 2001 From: hardfist Date: Mon, 18 May 2026 16:20:41 +0800 Subject: [PATCH 5/6] ci: show flamegraphs and base-vs-head benchmark diff --- .github/workflows/benchmark.yml | 96 +++++++++++++++++++++++++++++++-- benches/resolver_gungraun.rs | 13 ++++- 2 files changed, 103 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 3eeba986..cf74d800 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -23,6 +23,9 @@ jobs: benchmark: name: Benchmark (Gungraun) runs-on: ubuntu-latest + env: + GUNGRAUN_HOME: ${{ github.workspace }}/target/gungraun + GUNGRAUN_VALGRIND_BIN: /usr/bin/valgrind steps: - name: Checkout Branch uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 @@ -53,14 +56,99 @@ jobs: with: tool: gungraun-runner@0.18.2 - - name: Run gungraun benchmark + - name: Run gungraun baseline benchmark (PR base) + if: github.event_name == 'pull_request' timeout-minutes: 45 env: - GUNGRAUN_VALGRIND_BIN: /usr/bin/valgrind + BASE_SHA: ${{ github.event.pull_request.base.sha }} RUSTFLAGS: "-C debuginfo=1 -C strip=none -g" - run: cargo bench --bench resolver_gungraun -- --nocapture + run: | + set -euxo pipefail + git fetch --no-tags --depth=1 origin "${BASE_SHA}" + git worktree add /tmp/gungraun-pr-base "${BASE_SHA}" + pushd /tmp/gungraun-pr-base + pnpm install --dir benches --ignore-workspace + cargo bench --bench resolver_gungraun -- \ + --home="${GUNGRAUN_HOME}" \ + --save-baseline=pr_base \ + --save-summary=pretty-json + popd + git worktree remove /tmp/gungraun-pr-base --force - - name: Upload gungraun result files + - name: Run gungraun benchmark with baseline compare (PR head) + if: github.event_name == 'pull_request' + timeout-minutes: 45 + env: + RUSTFLAGS: "-C debuginfo=1 -C strip=none -g" + run: | + set -euxo pipefail + cargo bench --bench resolver_gungraun -- \ + --home="${GUNGRAUN_HOME}" \ + --baseline=pr_base \ + --save-summary=pretty-json \ + --output-format=json \ + > target/gungraun/benchmark-diff.ndjson + + - name: Run gungraun benchmark (push/manual) + if: github.event_name != 'pull_request' + timeout-minutes: 45 + env: + RUSTFLAGS: "-C debuginfo=1 -C strip=none -g" + run: | + set -euxo pipefail + cargo bench --bench resolver_gungraun -- \ + --home="${GUNGRAUN_HOME}" \ + --save-summary=pretty-json + + - name: Publish benchmark and commit diff summary + if: always() + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.sha }} + run: | + set -euo pipefail + + echo "## Gungraun Benchmark" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "- Base commit: \`${BASE_SHA}\`" >> "$GITHUB_STEP_SUMMARY" + echo "- Head commit: \`${HEAD_SHA}\`" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "### Benchmark Diff (Estimated Cycles)" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "| Benchmark | Base | Head | Diff |" >> "$GITHUB_STEP_SUMMARY" + echo "|---|---:|---:|---:|" >> "$GITHUB_STEP_SUMMARY" + + shopt -s nullglob + summary_files=(target/gungraun/rspack_resolver/resolver_gungraun/resolver_group/*/summary.json) + if [ "${#summary_files[@]}" -eq 0 ]; then + echo "| (no benchmark summary found) | - | - | - |" >> "$GITHUB_STEP_SUMMARY" + else + for f in "${summary_files[@]}"; do + jq -r ' + def n: (.Int // .Float // 0); + def m($k): .profiles[0].summaries.total.summary.Callgrind[$k]; + "| \(.id) | \((m("EstimatedCycles").metrics.Both[1] | n)) | \((m("EstimatedCycles").metrics.Both[0] | n)) | \((m("EstimatedCycles").diffs.diff_pct | tonumber) | if . > 0 then "+" + (.|tostring) else (.|tostring) end)% |" + ' "$f" >> "$GITHUB_STEP_SUMMARY" + done + fi + + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "### Code Diff (Base..Head)" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo '```text' >> "$GITHUB_STEP_SUMMARY" + git diff --stat --no-color "${BASE_SHA}..${HEAD_SHA}" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + else + echo "- Commit: \`${HEAD_SHA}\`" >> "$GITHUB_STEP_SUMMARY" + fi + + flamegraph_count="$(find target/gungraun -name '*.flamegraph*.svg' | wc -l | tr -d ' ')" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "Flamegraph SVG files: ${flamegraph_count}" >> "$GITHUB_STEP_SUMMARY" + + - name: Upload gungraun result files (including flamegraphs) if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: diff --git a/benches/resolver_gungraun.rs b/benches/resolver_gungraun.rs index 9750ae7b..bf73b313 100644 --- a/benches/resolver_gungraun.rs +++ b/benches/resolver_gungraun.rs @@ -1,4 +1,7 @@ -use gungraun::{binary_benchmark, binary_benchmark_group, main, BinaryBenchmarkConfig, Command}; +use gungraun::{ + binary_benchmark, binary_benchmark_group, main, BinaryBenchmarkConfig, Callgrind, Command, + FlamegraphConfig, FlamegraphKind, +}; #[binary_benchmark] #[bench::resolve_dependencies("deps")] @@ -13,6 +16,12 @@ fn bench_resolver(scenario: &str) -> Command { binary_benchmark_group!(name = resolver_group, benchmarks = bench_resolver); main!( - config = BinaryBenchmarkConfig::default().env_clear(false), + config = BinaryBenchmarkConfig::default().env_clear(false).tool( + Callgrind::default().flamegraph( + FlamegraphConfig::default() + .kind(FlamegraphKind::All) + .normalize_differential(true), + ), + ), binary_benchmark_groups = resolver_group ); From ce32bb95a58cafd40d99fc97b66649547b0d9d68 Mon Sep 17 00:00:00 2001 From: hardfist Date: Mon, 18 May 2026 16:24:58 +0800 Subject: [PATCH 6/6] ci: tolerate missing base benchmark target in gungraun --- .github/workflows/benchmark.yml | 80 ++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 22 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index cf74d800..8d261d0e 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -26,6 +26,7 @@ jobs: env: GUNGRAUN_HOME: ${{ github.workspace }}/target/gungraun GUNGRAUN_VALGRIND_BIN: /usr/bin/valgrind + GUNGRAUN_HAS_BASELINE: "0" steps: - name: Checkout Branch uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 @@ -63,17 +64,32 @@ jobs: BASE_SHA: ${{ github.event.pull_request.base.sha }} RUSTFLAGS: "-C debuginfo=1 -C strip=none -g" run: | - set -euxo pipefail + set -euo pipefail + WORKTREE_DIR=/tmp/gungraun-pr-base + cleanup() { + if [ -d "${WORKTREE_DIR}" ]; then + git worktree remove "${WORKTREE_DIR}" --force || true + fi + } + trap cleanup EXIT + + echo "GUNGRAUN_HAS_BASELINE=0" >> "$GITHUB_ENV" git fetch --no-tags --depth=1 origin "${BASE_SHA}" - git worktree add /tmp/gungraun-pr-base "${BASE_SHA}" - pushd /tmp/gungraun-pr-base + git worktree add "${WORKTREE_DIR}" "${BASE_SHA}" + pushd "${WORKTREE_DIR}" pnpm install --dir benches --ignore-workspace - cargo bench --bench resolver_gungraun -- \ - --home="${GUNGRAUN_HOME}" \ - --save-baseline=pr_base \ - --save-summary=pretty-json + + if cargo bench --bench resolver_gungraun --no-run >/dev/null 2>&1; then + cargo bench --bench resolver_gungraun -- \ + --home="${GUNGRAUN_HOME}" \ + --save-baseline=pr_base \ + --save-summary=pretty-json + echo "GUNGRAUN_HAS_BASELINE=1" >> "$GITHUB_ENV" + else + echo "Base commit ${BASE_SHA} has no resolver_gungraun bench target; skipping baseline compare." + fi + popd - git worktree remove /tmp/gungraun-pr-base --force - name: Run gungraun benchmark with baseline compare (PR head) if: github.event_name == 'pull_request' @@ -81,13 +97,20 @@ jobs: env: RUSTFLAGS: "-C debuginfo=1 -C strip=none -g" run: | - set -euxo pipefail - cargo bench --bench resolver_gungraun -- \ - --home="${GUNGRAUN_HOME}" \ - --baseline=pr_base \ - --save-summary=pretty-json \ - --output-format=json \ - > target/gungraun/benchmark-diff.ndjson + set -euo pipefail + mkdir -p target/gungraun + if [ "${GUNGRAUN_HAS_BASELINE:-0}" = "1" ]; then + cargo bench --bench resolver_gungraun -- \ + --home="${GUNGRAUN_HOME}" \ + --baseline=pr_base \ + --save-summary=pretty-json \ + --output-format=json \ + > target/gungraun/benchmark-diff.ndjson + else + cargo bench --bench resolver_gungraun -- \ + --home="${GUNGRAUN_HOME}" \ + --save-summary=pretty-json + fi - name: Run gungraun benchmark (push/manual) if: github.event_name != 'pull_request' @@ -114,6 +137,7 @@ jobs: if [ "${{ github.event_name }}" = "pull_request" ]; then echo "- Base commit: \`${BASE_SHA}\`" >> "$GITHUB_STEP_SUMMARY" echo "- Head commit: \`${HEAD_SHA}\`" >> "$GITHUB_STEP_SUMMARY" + echo "- Baseline compare: $([ "${GUNGRAUN_HAS_BASELINE:-0}" = "1" ] && echo "enabled" || echo "skipped (base has no resolver_gungraun target)")" >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" echo "### Benchmark Diff (Estimated Cycles)" >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" @@ -126,11 +150,19 @@ jobs: echo "| (no benchmark summary found) | - | - | - |" >> "$GITHUB_STEP_SUMMARY" else for f in "${summary_files[@]}"; do - jq -r ' - def n: (.Int // .Float // 0); - def m($k): .profiles[0].summaries.total.summary.Callgrind[$k]; - "| \(.id) | \((m("EstimatedCycles").metrics.Both[1] | n)) | \((m("EstimatedCycles").metrics.Both[0] | n)) | \((m("EstimatedCycles").diffs.diff_pct | tonumber) | if . > 0 then "+" + (.|tostring) else (.|tostring) end)% |" - ' "$f" >> "$GITHUB_STEP_SUMMARY" + if [ "${GUNGRAUN_HAS_BASELINE:-0}" = "1" ]; then + jq -r ' + def n: (.Int // .Float // 0); + def m($k): .profiles[0].summaries.total.summary.Callgrind[$k]; + "| \(.id) | \((m("EstimatedCycles").metrics.Both[1] | n)) | \((m("EstimatedCycles").metrics.Both[0] | n)) | \((m("EstimatedCycles").diffs.diff_pct | tonumber) | if . > 0 then "+" + (.|tostring) else (.|tostring) end)% |" + ' "$f" >> "$GITHUB_STEP_SUMMARY" + else + jq -r ' + def n: (.Int // .Float // 0); + def m($k): .profiles[0].summaries.total.summary.Callgrind[$k]; + "| \(.id) | - | \((m("EstimatedCycles").metrics.Both[0] | n)) | - |" + ' "$f" >> "$GITHUB_STEP_SUMMARY" + fi done fi @@ -144,9 +176,13 @@ jobs: echo "- Commit: \`${HEAD_SHA}\`" >> "$GITHUB_STEP_SUMMARY" fi - flamegraph_count="$(find target/gungraun -name '*.flamegraph*.svg' | wc -l | tr -d ' ')" echo "" >> "$GITHUB_STEP_SUMMARY" - echo "Flamegraph SVG files: ${flamegraph_count}" >> "$GITHUB_STEP_SUMMARY" + if [ -d target/gungraun ]; then + flamegraph_count="$(find target/gungraun -name '*.flamegraph*.svg' | wc -l | tr -d ' ')" + echo "Flamegraph SVG files: ${flamegraph_count}" >> "$GITHUB_STEP_SUMMARY" + else + echo "Flamegraph SVG files: 0 (target/gungraun not generated)" >> "$GITHUB_STEP_SUMMARY" + fi - name: Upload gungraun result files (including flamegraphs) if: always()