From cb6f353a64738ded0fc65a517a2e32183fdf0b9c Mon Sep 17 00:00:00 2001 From: Jan Gutsche Date: Fri, 17 Jul 2026 21:56:02 +0200 Subject: [PATCH 1/2] build: add production robot profile --- .github/workflows/build-benchmark.yml | 1 + pixi.toml | 7 ++++++- scripts/benchmark_build.py | 12 ++++++++++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-benchmark.yml b/.github/workflows/build-benchmark.yml index 44bedfb37..ba4a63e6b 100644 --- a/.github/workflows/build-benchmark.yml +++ b/.github/workflows/build-benchmark.yml @@ -41,6 +41,7 @@ jobs: - name: Measure clean and incremental builds run: >- python3 scripts/benchmark_build.py + --task build-robot-profile --output build-benchmark-${{ runner.arch }}.json - name: Show compiler cache statistics diff --git a/pixi.toml b/pixi.toml index 87d7caf97..6ef82f7a3 100644 --- a/pixi.toml +++ b/pixi.toml @@ -25,8 +25,13 @@ format = {cmd = "sh -c 'pre-commit run --all-files --hook-stage \"${PRE_COMMIT_S cmd = "colcon build --symlink-install --packages-skip zed_wrapper zed_components zed_msgs zed_ros2 --cmake-args -G Ninja -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_FLAGS='-Wall -Wextra -Werror -Wno-error=pedantic' -DCMAKE_CXX_FLAGS='-Wall -Wextra -Wpedantic -Werror' " description = "Builds all ROS 2 packages. Add --packages-select to build specific packages." +[tasks.build-robot-profile] +# Keep the package selection aligned with the default build so CI benchmark results remain comparable. +cmd = "colcon build --symlink-install --packages-skip zed_wrapper zed_components zed_msgs zed_ros2 --cmake-args -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_TESTING=OFF -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_FLAGS='-Wall -Wextra -Werror -Wno-error=pedantic' -DCMAKE_CXX_FLAGS='-Wall -Wextra -Wpedantic -Werror' " +description = "Builds the CI-compatible package set with the production robot profile." + [feature.robot.tasks.build] -cmd = "colcon build --symlink-install --cmake-args -G Ninja -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_FLAGS='-Wall -Wextra -Werror -Wno-error=pedantic' -DCMAKE_CXX_FLAGS='-Wall -Wextra -Wpedantic -Werror' " +cmd = "colcon build --symlink-install --cmake-args -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_TESTING=OFF -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_FLAGS='-Wall -Wextra -Werror -Wno-error=pedantic' -DCMAKE_CXX_FLAGS='-Wall -Wextra -Wpedantic -Werror' " description = "Builds all ROS 2 packages. Add --packages-select to build specific packages." [tasks.clean] diff --git a/scripts/benchmark_build.py b/scripts/benchmark_build.py index 755f9923b..4d8b62779 100644 --- a/scripts/benchmark_build.py +++ b/scripts/benchmark_build.py @@ -37,6 +37,7 @@ def append_github_summary(results: dict[str, object], summary_path: Path) -> Non summary.write("| --- | ---: | ---: |\n") summary.write(f"| Clean | {clean['duration_seconds']:.2f} s | {clean['return_code']} |\n") summary.write(f"| Incremental | {incremental['duration_seconds']:.2f} s | {incremental['return_code']} |\n\n") + summary.write(f"Pixi task: `{results['task']}`\n\n") summary.write(f"Touched source: `{results['touched_source']}`\n") @@ -53,6 +54,11 @@ def parse_args() -> argparse.Namespace: default=Path("build-benchmark.json"), help="JSON file that receives the benchmark results.", ) + parser.add_argument( + "--task", + default="build", + help="Pixi task used for both measured builds.", + ) parser.add_argument( "--source", type=Path, @@ -71,16 +77,18 @@ def main() -> int: ["pixi", "run", "-e", args.environment, "clean"], check=True, ) - clean_build = run_timed(["pixi", "run", "-e", args.environment, "build"]) + build_command = ["pixi", "run", "-e", args.environment, args.task] + clean_build = run_timed(build_command) # Updating the timestamp makes the build system reconsider one translation unit. # The file content stays unchanged, allowing compiler caches to reuse their output. os.utime(args.source) - incremental_build = run_timed(["pixi", "run", "-e", args.environment, "build"]) + incremental_build = run_timed(build_command) results = { "created_at": datetime.now(UTC).isoformat(), "environment": args.environment, + "task": args.task, "touched_source": str(args.source), "clean_build": clean_build, "incremental_build": incremental_build, From 50b6fec78a6f50eba8fb136e02d0d16119f88c34 Mon Sep 17 00:00:00 2001 From: Jan Gutsche Date: Sat, 18 Jul 2026 09:57:36 +0200 Subject: [PATCH 2/2] ci: repeat build optimization experiments --- .github/workflows/build-benchmark.yml | 21 +-- scripts/benchmark_build.py | 203 +++++++++++++++++++++++++- 2 files changed, 204 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build-benchmark.yml b/.github/workflows/build-benchmark.yml index ba4a63e6b..59476d2ec 100644 --- a/.github/workflows/build-benchmark.yml +++ b/.github/workflows/build-benchmark.yml @@ -1,4 +1,5 @@ name: Build benchmark +run-name: Repeated build benchmark (3x) on: pull_request: @@ -11,8 +12,8 @@ jobs: matrix: os: [ubuntu-latest, ubuntu-24.04-arm] runs-on: ${{ matrix.os }} + timeout-minutes: 300 env: - CCACHE_DIR: ${{ github.workspace }}/.ccache PSModulePath: "" steps: @@ -29,25 +30,13 @@ jobs: cache: true cache-key: pixi-${{ hashFiles('pixi.toml', 'pixi.lock') }}- - - name: Restore compiler cache - uses: actions/cache@v5 - with: - path: ${{ env.CCACHE_DIR }} - key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('pixi.lock') }}-${{ github.sha }} - restore-keys: | - ccache-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('pixi.lock') }}- - ccache-${{ runner.os }}-${{ runner.arch }}- - - - name: Measure clean and incremental builds + - name: Measure all build configurations three times run: >- python3 scripts/benchmark_build.py - --task build-robot-profile + --suite + --repetitions 3 --output build-benchmark-${{ runner.arch }}.json - - name: Show compiler cache statistics - if: always() - run: pixi run -e default ccache --show-stats - - name: Upload benchmark results if: always() uses: actions/upload-artifact@v7 diff --git a/scripts/benchmark_build.py b/scripts/benchmark_build.py index 4d8b62779..bdda1b2cb 100644 --- a/scripts/benchmark_build.py +++ b/scripts/benchmark_build.py @@ -5,18 +5,66 @@ import argparse import json import os +import shutil import subprocess import time +from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path +from statistics import fmean DEFAULT_SOURCE = Path("src/bitbots_motion/bitbots_head_mover/src/move_head.cpp") +PACKAGES_TO_SKIP = ("zed_wrapper", "zed_components", "zed_msgs", "zed_ros2") +COMMON_CMAKE_ARGS = ( + "-DCMAKE_C_FLAGS=-Wall -Wextra -Werror -Wno-error=pedantic", + "-DCMAKE_CXX_FLAGS=-Wall -Wextra -Wpedantic -Werror", +) -def run_timed(command: list[str]) -> dict[str, float | int | list[str]]: +@dataclass(frozen=True) +class Experiment: + """One build-system configuration in the repeated benchmark suite.""" + + name: str + label: str + cmake_args: tuple[str, ...] + uses_ccache: bool = False + + +EXPERIMENTS = ( + Experiment("baseline", "Unix Makefiles", ("-G", "Unix Makefiles")), + Experiment("ninja", "Ninja", ("-G", "Ninja")), + Experiment( + "ccache", + "Ninja + ccache", + ( + "-G", + "Ninja", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + ), + uses_ccache=True, + ), + Experiment( + "robot-profile", + "Robot profile", + ( + "-G", + "Ninja", + "-DCMAKE_BUILD_TYPE=RelWithDebInfo", + "-DBUILD_TESTING=OFF", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + ), + uses_ccache=True, + ), +) + + +def run_timed(command: list[str], env: dict[str, str] | None = None) -> dict[str, float | int | list[str]]: """Run a command and return its duration and exit status.""" start = time.perf_counter() - result = subprocess.run(command, check=False) + result = subprocess.run(command, check=False, env=env) return { "command": command, "duration_seconds": time.perf_counter() - start, @@ -24,7 +72,7 @@ def run_timed(command: list[str]) -> dict[str, float | int | list[str]]: } -def append_github_summary(results: dict[str, object], summary_path: Path) -> None: +def append_single_summary(results: dict[str, object], summary_path: Path) -> None: """Append the benchmark results to a GitHub Actions step summary.""" clean = results["clean_build"] incremental = results["incremental_build"] @@ -41,6 +89,102 @@ def append_github_summary(results: dict[str, object], summary_path: Path) -> Non summary.write(f"Touched source: `{results['touched_source']}`\n") +def append_suite_summary(results: dict[str, object], summary_path: Path) -> None: + """Append all repeated experiment results to a GitHub Actions step summary.""" + experiments = results["experiments"] + assert isinstance(experiments, list) + + with summary_path.open("a", encoding="utf-8") as summary: + summary.write("## Repeated build benchmark\n\n") + summary.write("| Experiment | Repetition | Clean | Incremental |\n") + summary.write("| --- | ---: | ---: | ---: |\n") + for experiment in experiments: + assert isinstance(experiment, dict) + runs = experiment["runs"] + assert isinstance(runs, list) + for run in runs: + assert isinstance(run, dict) + clean = run["clean_build"] + incremental = run["incremental_build"] + assert isinstance(clean, dict) + assert isinstance(incremental, dict) + summary.write( + f"| {experiment['label']} | {run['repetition']} " + f"| {clean['duration_seconds']:.2f} s " + f"| {incremental['duration_seconds']:.2f} s |\n" + ) + averages = experiment["averages"] + assert isinstance(averages, dict) + summary.write( + f"| **{experiment['label']}** | **mean** " + f"| **{averages['clean_seconds']:.2f} s** " + f"| **{averages['incremental_seconds']:.2f} s** |\n" + ) + summary.write(f"\nTouched source: `{results['touched_source']}`\n") + + +def experiment_build_command(environment: str, experiment: Experiment) -> list[str]: + """Construct the exact colcon command for an experiment.""" + return [ + "pixi", + "run", + "-e", + environment, + "colcon", + "build", + "--symlink-install", + "--packages-skip", + *PACKAGES_TO_SKIP, + "--cmake-args", + *experiment.cmake_args, + *COMMON_CMAKE_ARGS, + ] + + +def run_experiment( + experiment: Experiment, + repetitions: int, + environment: str, + source: Path, + cache_root: Path, +) -> dict[str, object]: + """Run one configuration repeatedly while retaining its isolated compiler cache.""" + command = experiment_build_command(environment, experiment) + experiment_environment = os.environ.copy() + if experiment.uses_ccache: + experiment_environment["CCACHE_DIR"] = str(cache_root / experiment.name) + + runs = [] + for repetition in range(1, repetitions + 1): + subprocess.run( + ["pixi", "run", "-e", environment, "clean"], + check=True, + env=experiment_environment, + ) + clean_build = run_timed(command, experiment_environment) + os.utime(source) + incremental_build = run_timed(command, experiment_environment) + runs.append( + { + "repetition": repetition, + "clean_build": clean_build, + "incremental_build": incremental_build, + } + ) + + return { + "name": experiment.name, + "label": experiment.label, + "uses_ccache": experiment.uses_ccache, + "command": command, + "runs": runs, + "averages": { + "clean_seconds": fmean(run["clean_build"]["duration_seconds"] for run in runs), + "incremental_seconds": fmean(run["incremental_build"]["duration_seconds"] for run in runs), + }, + } + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -59,6 +203,17 @@ def parse_args() -> argparse.Namespace: default="build", help="Pixi task used for both measured builds.", ) + parser.add_argument( + "--suite", + action="store_true", + help="Run all build-system experiments instead of one Pixi task.", + ) + parser.add_argument( + "--repetitions", + type=int, + default=1, + help="Number of clean and incremental measurements per suite experiment.", + ) parser.add_argument( "--source", type=Path, @@ -72,6 +227,46 @@ def main() -> int: args = parse_args() if not args.source.is_file(): raise FileNotFoundError(f"Incremental benchmark source does not exist: {args.source}") + if args.repetitions < 1: + raise ValueError("Repetitions must be at least one") + + if args.suite: + runner_temp = os.environ.get("RUNNER_TEMP") + cache_root = ( + Path(runner_temp) / "bitbots-build-benchmark-ccache" + if runner_temp + else Path(".ccache/build-benchmark-suite") + ) + shutil.rmtree(cache_root, ignore_errors=True) + suite_results = [ + run_experiment( + experiment, + args.repetitions, + args.environment, + args.source, + cache_root, + ) + for experiment in EXPERIMENTS + ] + results = { + "created_at": datetime.now(UTC).isoformat(), + "environment": args.environment, + "repetitions": args.repetitions, + "touched_source": str(args.source), + "experiments": suite_results, + } + args.output.write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8") + + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + append_suite_summary(results, Path(summary_path)) + + return max( + build["return_code"] + for experiment in suite_results + for run in experiment["runs"] + for build in (run["clean_build"], run["incremental_build"]) + ) subprocess.run( ["pixi", "run", "-e", args.environment, "clean"], @@ -97,7 +292,7 @@ def main() -> int: summary_path = os.environ.get("GITHUB_STEP_SUMMARY") if summary_path: - append_github_summary(results, Path(summary_path)) + append_single_summary(results, Path(summary_path)) return max(clean_build["return_code"], incremental_build["return_code"])