From c71bf57417b1b68284879b72bc5d46b65a082d70 Mon Sep 17 00:00:00 2001 From: Konrad Kleine Date: Mon, 26 Aug 2024 17:19:02 +0000 Subject: [PATCH 1/9] compare the system llvm against "big-merge" and "pgo" --- scripts/perf/Containerfile | 42 +++++++++++++ scripts/perf/Makefile | 4 ++ scripts/perf/README.md | 27 +++++++++ scripts/perf/entrypoint.sh | 118 +++++++++++++++++++++++++++++++++++++ 4 files changed, 191 insertions(+) create mode 100644 scripts/perf/Containerfile create mode 100644 scripts/perf/Makefile create mode 100644 scripts/perf/README.md create mode 100755 scripts/perf/entrypoint.sh diff --git a/scripts/perf/Containerfile b/scripts/perf/Containerfile new file mode 100644 index 00000000..ee227490 --- /dev/null +++ b/scripts/perf/Containerfile @@ -0,0 +1,42 @@ +FROM fedora:40 +LABEL description="Test compilers with llvm-test-suite" + +USER root +WORKDIR /root + +# Install deps to run test-suite +RUN dnf install -y \ + cmake \ + fedora-packager \ + git \ + python3-pip \ + python3-virtualenv \ + python3-lit \ + ninja-build \ + which \ + coreutils \ + tcl \ + tcl-devel \ + tcl-tclreadline \ + tcl-tclxml-devel \ + tcl-tclxml \ + tcl-zlib \ + tcl-thread-devel + +RUN virtualenv ~/mysandbox +RUN source ~/mysandbox/bin/activate \ + && pip install \ + pandas \ + scipy + +# Clone test suite (in correct version for installed clang version) +# See https://llvm.org/docs/TestSuiteGuide.html +# RUN export VERSION=`clang --version | grep -ioP 'clang version\s\K[0-9\.]+'` \ +# && git clone --depth=1 --branch llvmorg-${VERSION} https://github.com/llvm/llvm-test-suite.git test-suite +RUN git clone --depth=1 https://github.com/llvm/llvm-test-suite.git test-suite + +RUN dnf install -y 'dnf-command(copr)' perf + +COPY entrypoint.sh /root/entrypoint.sh +USER root +ENTRYPOINT [ "/root/entrypoint.sh" ] diff --git a/scripts/perf/Makefile b/scripts/perf/Makefile new file mode 100644 index 00000000..94f55176 --- /dev/null +++ b/scripts/perf/Makefile @@ -0,0 +1,4 @@ +.PHONY: all +all: + podman build -t evaluation . + podman run -it evaluation diff --git a/scripts/perf/README.md b/scripts/perf/README.md new file mode 100644 index 00000000..ea07c19e --- /dev/null +++ b/scripts/perf/README.md @@ -0,0 +1,27 @@ +# README + +This container setup allows you to compare the system llvm against "big-merge" and "pgo". + +# How to + +Just run `make` to build and run the container image. It takes a long time to complete. + +Then you'll be promted to a terminal in the container where you'll find these files: + +``` +~/results-system-vs-pgo.txt +~/results-system-vs-big-merge.txt +~/results-big-merge-vs-pgo.txt +``` + +The names speak for themselves. + +## How to change to OS + +If you want to change the version of the operating system, go to `Containerfile` and change the line that looks like this: `FROM fedora:40`. Change it to `FROM fedora:41` or something else. + +Then run `make` again. + +## How to change the date for which to compare results? + +Go to `entrypoint.sh` and change the line that defines `yyyymmdd` to the year-month-date of your liking. diff --git a/scripts/perf/entrypoint.sh b/scripts/perf/entrypoint.sh new file mode 100755 index 00000000..49b6b6fb --- /dev/null +++ b/scripts/perf/entrypoint.sh @@ -0,0 +1,118 @@ +#!/usr/bin/bash + +set -x + +# Source the python environment with required packages +source ~/mysandbox/bin/activate + +function configure_build_run { + # Configure the test suite + cmake \ + -DCMAKE_GENERATOR=Ninja \ + -DCMAKE_C_COMPILER=/usr/bin/clang \ + -DCMAKE_CXX_COMPILER=/usr/bin/clang++ \ + -C~/test-suite/cmake/caches/O3.cmake \ + ~/test-suite + + # Build the test-suite + ninja -j30 + + # Run the tests with lit: + lit -j1 -v -o results.json . || true +} + +# Query version information for given day +yyyymmdd=20240825 +git_rev=$(curl -sL https://github.com/fedora-llvm-team/llvm-snapshots/releases/download/snapshot-version-sync/llvm-git-revision-${yyyymmdd}.txt) +git_rev_short="${git_rev:0:14}" +llvm_release=$(curl -sL https://github.com/fedora-llvm-team/llvm-snapshots/releases/download/snapshot-version-sync/llvm-release-${yyyymmdd}.txt) +rpm_suffix="${llvm_release}~pre${yyyymmdd}.g${git_rev_short}" + +echo "git_rev=$git_rev" +echo "git_rev_short=$git_rev_short" +echo "llvm_release=$llvm_release" +echo "rpm_suffix=$rpm_suffix" + +###################################################################################### +# PGO +###################################################################################### + +# Install and enable the repository that provides the PGO LLVM Toolchain +# See https://llvm.org/docs/HowToBuildWithPGO.html#building-clang-with-pgo +dnf copr enable -y @fedora-llvm-team/llvm-snapshots-pgo-${yyyymmdd} +dnf install -y \ + clang-${rpm_suffix} \ + clang-${rpm_suffix} \ + clang-libs-${rpm_suffix} \ + clang-resource-filesystem-${rpm_suffix} \ + llvm-${rpm_suffix} \ + llvm-libs-${rpm_suffix} + +mkdir -pv ~/pgo +cd ~/pgo + +configure_build_run +# Remove packages from that PGO repo and the repo itself +dnf -y repository-packages copr:copr.fedorainfracloud.org:group_fedora-llvm-team:llvm-snapshots-pgo-${yyyymmdd} remove +dnf -y copr remove @fedora-llvm-team/llvm-snapshots-pgo-${yyyymmdd} + +###################################################################################### +# big-merge +###################################################################################### + +# Install and enable the repository that provides the big-merge LLVM Toolchain +dnf copr enable -y @fedora-llvm-team/llvm-snapshots-big-merge-${yyyymmdd} +dnf install -y \ + clang-${rpm_suffix} \ + clang-${rpm_suffix} \ + clang-libs-${rpm_suffix} \ + clang-resource-filesystem-${rpm_suffix} \ + llvm-${rpm_suffix} \ + llvm-libs-${rpm_suffix} + +mkdir -pv ~/big-merge +cd ~/big-merge + +configure_build_run +# Remove packages from that big-merge repo and the repo itself +dnf -y repository-packages copr:copr.fedorainfracloud.org:group_fedora-llvm-team:llvm-snapshots-big-merge-${yyyymmdd} remove +dnf -y copr remove @fedora-llvm-team/llvm-snapshots-big-merge-${yyyymmdd} + +###################################################################################### +# system llvm +###################################################################################### + +# Build with regular clang +dnf install -y clang clang-libs clang-resource-filesystem llvm llvm-libs +mkdir -pv ~/system +cd ~/system + +configure_build_run + +system_llvm_release=$(clang --version | grep -Po '[0-9]+\.[0-9]+\.[0-9]' | head -n1) + +/root/test-suite/utils/compare.py \ + --metric exec_time \ + --metric compile_time \ + --metric link_time \ + --lhs-name ${system_llvm_release} \ + --rhs-name pgo-${rpm_suffix} \ + ~/system/results.json vs ~/pgo/results.json > ~/results-system-vs-pgo.txt || true + +/root/test-suite/utils/compare.py \ + --metric exec_time \ + --metric compile_time \ + --metric link_time \ + --lhs-name ${system_llvm_release} \ + --rhs-name big-merge-${rpm_suffix} \ + ~/system/results.json vs ~/big-merge/results.json > ~/results-system-vs-big-merge.txt || true + +/root/test-suite/utils/compare.py \ + --metric exec_time \ + --metric compile_time \ + --metric link_time \ + --lhs-name big-merge-${rpm_suffix} \ + --rhs-name pgo-${rpm_suffix} \ + ~/big-merge/results.json vs ~/pgo/results.json > ~/results-big-merge-vs-pgo.txt || true + +bash From a748d92ef96ac8a727f3f35fd2d65613061ba415 Mon Sep 17 00:00:00 2001 From: Konrad Kleine Date: Tue, 27 Aug 2024 14:47:45 +0200 Subject: [PATCH 2/9] Update entrypoint.sh Make result txt files easier to read with smaller lhs and rhs column widths. --- scripts/perf/entrypoint.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/perf/entrypoint.sh b/scripts/perf/entrypoint.sh index 49b6b6fb..ef971871 100755 --- a/scripts/perf/entrypoint.sh +++ b/scripts/perf/entrypoint.sh @@ -96,7 +96,7 @@ system_llvm_release=$(clang --version | grep -Po '[0-9]+\.[0-9]+\.[0-9]' | head --metric compile_time \ --metric link_time \ --lhs-name ${system_llvm_release} \ - --rhs-name pgo-${rpm_suffix} \ + --rhs-name pgo \ ~/system/results.json vs ~/pgo/results.json > ~/results-system-vs-pgo.txt || true /root/test-suite/utils/compare.py \ @@ -104,15 +104,15 @@ system_llvm_release=$(clang --version | grep -Po '[0-9]+\.[0-9]+\.[0-9]' | head --metric compile_time \ --metric link_time \ --lhs-name ${system_llvm_release} \ - --rhs-name big-merge-${rpm_suffix} \ + --rhs-name big-merge} \ ~/system/results.json vs ~/big-merge/results.json > ~/results-system-vs-big-merge.txt || true /root/test-suite/utils/compare.py \ --metric exec_time \ --metric compile_time \ --metric link_time \ - --lhs-name big-merge-${rpm_suffix} \ - --rhs-name pgo-${rpm_suffix} \ + --lhs-name big-merge \ + --rhs-name pgo \ ~/big-merge/results.json vs ~/pgo/results.json > ~/results-big-merge-vs-pgo.txt || true bash From 94d91b8cd373b07dd0f047416cf1a03388ab7d5b Mon Sep 17 00:00:00 2001 From: Konrad Kleine Date: Mon, 26 Aug 2024 19:00:48 +0000 Subject: [PATCH 3/9] Install from copr repo --- scripts/perf/entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/perf/entrypoint.sh b/scripts/perf/entrypoint.sh index ef971871..7aff1240 100755 --- a/scripts/perf/entrypoint.sh +++ b/scripts/perf/entrypoint.sh @@ -40,7 +40,7 @@ echo "rpm_suffix=$rpm_suffix" # Install and enable the repository that provides the PGO LLVM Toolchain # See https://llvm.org/docs/HowToBuildWithPGO.html#building-clang-with-pgo dnf copr enable -y @fedora-llvm-team/llvm-snapshots-pgo-${yyyymmdd} -dnf install -y \ +dnf -y repository-packages copr:copr.fedorainfracloud.org:group_fedora-llvm-team:llvm-snapshots-pgo-${yyyymmdd} install \ clang-${rpm_suffix} \ clang-${rpm_suffix} \ clang-libs-${rpm_suffix} \ @@ -62,7 +62,7 @@ dnf -y copr remove @fedora-llvm-team/llvm-snapshots-pgo-${yyyymmdd} # Install and enable the repository that provides the big-merge LLVM Toolchain dnf copr enable -y @fedora-llvm-team/llvm-snapshots-big-merge-${yyyymmdd} -dnf install -y \ +dnf -y repository-packages copr:copr.fedorainfracloud.org:group_fedora-llvm-team:llvm-snapshots-big-merge-${yyyymmdd} install \ clang-${rpm_suffix} \ clang-${rpm_suffix} \ clang-libs-${rpm_suffix} \ From d921a7db698a3722aa4b90fda9ce1eae2bcd2a4f Mon Sep 17 00:00:00 2001 From: Konrad Kleine Date: Mon, 9 Sep 2024 15:54:16 +0000 Subject: [PATCH 4/9] Prepare for testing with F41 with DNF5 --- scripts/perf/Containerfile | 13 ++++++++----- scripts/perf/Makefile | 2 +- scripts/perf/entrypoint.sh | 30 ++++++++++++++++++++---------- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/scripts/perf/Containerfile b/scripts/perf/Containerfile index ee227490..3c00f67e 100644 --- a/scripts/perf/Containerfile +++ b/scripts/perf/Containerfile @@ -1,4 +1,4 @@ -FROM fedora:40 +FROM fedora:41 LABEL description="Test compilers with llvm-test-suite" USER root @@ -24,10 +24,13 @@ RUN dnf install -y \ tcl-thread-devel RUN virtualenv ~/mysandbox -RUN source ~/mysandbox/bin/activate \ - && pip install \ - pandas \ - scipy +RUN dnf install -y clang +RUN dnf install -y python3-pandas python3-scipy +RUN dnf install -y jq envsubst +#RUN source ~/mysandbox/bin/activate \ +# && pip install \ +# pandas \ +# scipy # Clone test suite (in correct version for installed clang version) # See https://llvm.org/docs/TestSuiteGuide.html diff --git a/scripts/perf/Makefile b/scripts/perf/Makefile index 94f55176..f0572d62 100644 --- a/scripts/perf/Makefile +++ b/scripts/perf/Makefile @@ -1,4 +1,4 @@ .PHONY: all all: podman build -t evaluation . - podman run -it evaluation + podman run -it --rm evaluation diff --git a/scripts/perf/entrypoint.sh b/scripts/perf/entrypoint.sh index 7aff1240..9e86ec43 100755 --- a/scripts/perf/entrypoint.sh +++ b/scripts/perf/entrypoint.sh @@ -1,9 +1,10 @@ #!/usr/bin/bash set -x +set -e -# Source the python environment with required packages -source ~/mysandbox/bin/activate +## Source the python environment with required packages +#source ~/mysandbox/bin/activate function configure_build_run { # Configure the test suite @@ -15,14 +16,14 @@ function configure_build_run { ~/test-suite # Build the test-suite - ninja -j30 + ninja # Run the tests with lit: lit -j1 -v -o results.json . || true } # Query version information for given day -yyyymmdd=20240825 +yyyymmdd=20240909 git_rev=$(curl -sL https://github.com/fedora-llvm-team/llvm-snapshots/releases/download/snapshot-version-sync/llvm-git-revision-${yyyymmdd}.txt) git_rev_short="${git_rev:0:14}" llvm_release=$(curl -sL https://github.com/fedora-llvm-team/llvm-snapshots/releases/download/snapshot-version-sync/llvm-release-${yyyymmdd}.txt) @@ -40,7 +41,10 @@ echo "rpm_suffix=$rpm_suffix" # Install and enable the repository that provides the PGO LLVM Toolchain # See https://llvm.org/docs/HowToBuildWithPGO.html#building-clang-with-pgo dnf copr enable -y @fedora-llvm-team/llvm-snapshots-pgo-${yyyymmdd} -dnf -y repository-packages copr:copr.fedorainfracloud.org:group_fedora-llvm-team:llvm-snapshots-pgo-${yyyymmdd} install \ +repo_file=$(dnf repoinfo --json *llvm-snapshots-pgo* | jq -r ".[0].repo_file_path") +distname=$(rpm --eval "%{?fedora:fedora}%{?rhel:rhel}") envsubst '$distname' < $repo_file > /tmp/new_repo_file +cat /tmp/new_repo_file > $repo_file +dnf -y install \ clang-${rpm_suffix} \ clang-${rpm_suffix} \ clang-libs-${rpm_suffix} \ @@ -52,9 +56,11 @@ mkdir -pv ~/pgo cd ~/pgo configure_build_run + # Remove packages from that PGO repo and the repo itself -dnf -y repository-packages copr:copr.fedorainfracloud.org:group_fedora-llvm-team:llvm-snapshots-pgo-${yyyymmdd} remove -dnf -y copr remove @fedora-llvm-team/llvm-snapshots-pgo-${yyyymmdd} +repo_pkgs_installed=$(dnf repoquery --installed --queryformat ' %{name} %{from_repo} ' | grep -Po "[^ ]+ [^ ]+llvm-snapshots-pgo" | awk '{print $1}') +dnf -y remove $repo_pkgs_installed; +dnf copr disable -y @fedora-llvm-team/llvm-snapshots-pgo-${yyyymmdd} ###################################################################################### # big-merge @@ -62,7 +68,10 @@ dnf -y copr remove @fedora-llvm-team/llvm-snapshots-pgo-${yyyymmdd} # Install and enable the repository that provides the big-merge LLVM Toolchain dnf copr enable -y @fedora-llvm-team/llvm-snapshots-big-merge-${yyyymmdd} -dnf -y repository-packages copr:copr.fedorainfracloud.org:group_fedora-llvm-team:llvm-snapshots-big-merge-${yyyymmdd} install \ +repo_file=$(dnf repoinfo --json *llvm-snapshots-big-merge* | jq -r ".[0].repo_file_path") +distname=$(rpm --eval "%{?fedora:fedora}%{?rhel:rhel}") envsubst '$distname' < $repo_file > /tmp/new_repo_file +cat /tmp/new_repo_file > $repo_file +dnf -y install \ clang-${rpm_suffix} \ clang-${rpm_suffix} \ clang-libs-${rpm_suffix} \ @@ -75,8 +84,9 @@ cd ~/big-merge configure_build_run # Remove packages from that big-merge repo and the repo itself -dnf -y repository-packages copr:copr.fedorainfracloud.org:group_fedora-llvm-team:llvm-snapshots-big-merge-${yyyymmdd} remove -dnf -y copr remove @fedora-llvm-team/llvm-snapshots-big-merge-${yyyymmdd} +repo_pkgs_installed=$(dnf repoquery --installed --queryformat ' %{name} %{from_repo} ' | grep -Po "[^ ]+ [^ ]+llvm-snapshots-big-merge" | awk '{print $1}') +dnf -y remove $repo_pkgs_installed; +dnf copr disable -y @fedora-llvm-team/llvm-snapshots-big-merge-${yyyymmdd} ###################################################################################### # system llvm From 27327de82cb0b34a7f98464e974c350b0924d81a Mon Sep 17 00:00:00 2001 From: Konrad Kleine Date: Thu, 12 Sep 2024 07:59:46 +0000 Subject: [PATCH 5/9] Use perf --- scripts/perf/entrypoint.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/perf/entrypoint.sh b/scripts/perf/entrypoint.sh index 9e86ec43..6c6e022c 100755 --- a/scripts/perf/entrypoint.sh +++ b/scripts/perf/entrypoint.sh @@ -12,6 +12,9 @@ function configure_build_run { -DCMAKE_GENERATOR=Ninja \ -DCMAKE_C_COMPILER=/usr/bin/clang \ -DCMAKE_CXX_COMPILER=/usr/bin/clang++ \ + -DTEST_SUITE_BENCHMARKING_ONLY=ON \ + -DTEST_SUITE_COLLECT_STATS=ON \ + -DTEST_SUITE_USE_PERF=ON \ -C~/test-suite/cmake/caches/O3.cmake \ ~/test-suite @@ -23,7 +26,7 @@ function configure_build_run { } # Query version information for given day -yyyymmdd=20240909 +yyyymmdd=20240911 git_rev=$(curl -sL https://github.com/fedora-llvm-team/llvm-snapshots/releases/download/snapshot-version-sync/llvm-git-revision-${yyyymmdd}.txt) git_rev_short="${git_rev:0:14}" llvm_release=$(curl -sL https://github.com/fedora-llvm-team/llvm-snapshots/releases/download/snapshot-version-sync/llvm-release-${yyyymmdd}.txt) From 6ba22df792cbd3dd7358075b3ff703c7eb669c2b Mon Sep 17 00:00:00 2001 From: Konrad Kleine Date: Thu, 12 Sep 2024 08:02:53 +0000 Subject: [PATCH 6/9] Cleanup --- scripts/perf/Containerfile | 31 +++++++++++++++---------------- scripts/perf/Makefile | 8 +++++++- scripts/perf/entrypoint.sh | 3 --- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/scripts/perf/Containerfile b/scripts/perf/Containerfile index 3c00f67e..8186cafc 100644 --- a/scripts/perf/Containerfile +++ b/scripts/perf/Containerfile @@ -6,31 +6,30 @@ WORKDIR /root # Install deps to run test-suite RUN dnf install -y \ + clang \ cmake \ + coreutils \ + envsubst \ fedora-packager \ git \ + jq \ + jq \ + ninja-build \ + perf + python3-lit \ + python3-pandas \ python3-pip \ + python3-scipy + python3-setuptools \ python3-virtualenv \ - python3-lit \ - ninja-build \ - which \ - coreutils \ tcl \ tcl-devel \ tcl-tclreadline \ - tcl-tclxml-devel \ tcl-tclxml \ + tcl-tclxml-devel \ + tcl-thread-devel \ tcl-zlib \ - tcl-thread-devel - -RUN virtualenv ~/mysandbox -RUN dnf install -y clang -RUN dnf install -y python3-pandas python3-scipy -RUN dnf install -y jq envsubst -#RUN source ~/mysandbox/bin/activate \ -# && pip install \ -# pandas \ -# scipy + which \ # Clone test suite (in correct version for installed clang version) # See https://llvm.org/docs/TestSuiteGuide.html @@ -38,7 +37,7 @@ RUN dnf install -y jq envsubst # && git clone --depth=1 --branch llvmorg-${VERSION} https://github.com/llvm/llvm-test-suite.git test-suite RUN git clone --depth=1 https://github.com/llvm/llvm-test-suite.git test-suite -RUN dnf install -y 'dnf-command(copr)' perf +RUN dnf install -y 'dnf-command(copr)' COPY entrypoint.sh /root/entrypoint.sh USER root diff --git a/scripts/perf/Makefile b/scripts/perf/Makefile index f0572d62..2e5f7932 100644 --- a/scripts/perf/Makefile +++ b/scripts/perf/Makefile @@ -1,4 +1,10 @@ .PHONY: all -all: +all: setup run + +.PHONY: setup +setup: podman build -t evaluation . + +.PHONY: run +run: podman run -it --rm evaluation diff --git a/scripts/perf/entrypoint.sh b/scripts/perf/entrypoint.sh index 6c6e022c..13336b0f 100755 --- a/scripts/perf/entrypoint.sh +++ b/scripts/perf/entrypoint.sh @@ -3,9 +3,6 @@ set -x set -e -## Source the python environment with required packages -#source ~/mysandbox/bin/activate - function configure_build_run { # Configure the test suite cmake \ From dbe672fa4b8dc7aab24df67919c4e3ecb0505ac9 Mon Sep 17 00:00:00 2001 From: Konrad Kleine Date: Thu, 12 Sep 2024 08:20:27 +0000 Subject: [PATCH 7/9] Fixup --- scripts/perf/Containerfile | 6 +++--- scripts/perf/entrypoint.sh | 8 +++----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/scripts/perf/Containerfile b/scripts/perf/Containerfile index 8186cafc..30021fa5 100644 --- a/scripts/perf/Containerfile +++ b/scripts/perf/Containerfile @@ -15,11 +15,11 @@ RUN dnf install -y \ jq \ jq \ ninja-build \ - perf + perf \ python3-lit \ python3-pandas \ python3-pip \ - python3-scipy + python3-scipy \ python3-setuptools \ python3-virtualenv \ tcl \ @@ -29,7 +29,7 @@ RUN dnf install -y \ tcl-tclxml-devel \ tcl-thread-devel \ tcl-zlib \ - which \ + which # Clone test suite (in correct version for installed clang version) # See https://llvm.org/docs/TestSuiteGuide.html diff --git a/scripts/perf/entrypoint.sh b/scripts/perf/entrypoint.sh index 13336b0f..ef572d19 100755 --- a/scripts/perf/entrypoint.sh +++ b/scripts/perf/entrypoint.sh @@ -12,14 +12,15 @@ function configure_build_run { -DTEST_SUITE_BENCHMARKING_ONLY=ON \ -DTEST_SUITE_COLLECT_STATS=ON \ -DTEST_SUITE_USE_PERF=ON \ + -DTEST_SUITE_RUN_BENCHMARKS=OFF \ -C~/test-suite/cmake/caches/O3.cmake \ ~/test-suite # Build the test-suite - ninja + ninja -j1 # Run the tests with lit: - lit -j1 -v -o results.json . || true + lit -v -o results.json . || true } # Query version information for given day @@ -102,7 +103,6 @@ configure_build_run system_llvm_release=$(clang --version | grep -Po '[0-9]+\.[0-9]+\.[0-9]' | head -n1) /root/test-suite/utils/compare.py \ - --metric exec_time \ --metric compile_time \ --metric link_time \ --lhs-name ${system_llvm_release} \ @@ -110,7 +110,6 @@ system_llvm_release=$(clang --version | grep -Po '[0-9]+\.[0-9]+\.[0-9]' | head ~/system/results.json vs ~/pgo/results.json > ~/results-system-vs-pgo.txt || true /root/test-suite/utils/compare.py \ - --metric exec_time \ --metric compile_time \ --metric link_time \ --lhs-name ${system_llvm_release} \ @@ -118,7 +117,6 @@ system_llvm_release=$(clang --version | grep -Po '[0-9]+\.[0-9]+\.[0-9]' | head ~/system/results.json vs ~/big-merge/results.json > ~/results-system-vs-big-merge.txt || true /root/test-suite/utils/compare.py \ - --metric exec_time \ --metric compile_time \ --metric link_time \ --lhs-name big-merge \ From b4f4a2080936927f0f2ce3d2a4155baf2c5e27b3 Mon Sep 17 00:00:00 2001 From: Konrad Kleine Date: Thu, 12 Sep 2024 08:29:30 +0000 Subject: [PATCH 8/9] Use CTMark and drop link time --- scripts/perf/entrypoint.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/scripts/perf/entrypoint.sh b/scripts/perf/entrypoint.sh index ef572d19..bc271f9e 100755 --- a/scripts/perf/entrypoint.sh +++ b/scripts/perf/entrypoint.sh @@ -12,6 +12,7 @@ function configure_build_run { -DTEST_SUITE_BENCHMARKING_ONLY=ON \ -DTEST_SUITE_COLLECT_STATS=ON \ -DTEST_SUITE_USE_PERF=ON \ + -DTEST_SUITE_SUBDIRS=CTMark \ -DTEST_SUITE_RUN_BENCHMARKS=OFF \ -C~/test-suite/cmake/caches/O3.cmake \ ~/test-suite @@ -104,23 +105,20 @@ system_llvm_release=$(clang --version | grep -Po '[0-9]+\.[0-9]+\.[0-9]' | head /root/test-suite/utils/compare.py \ --metric compile_time \ - --metric link_time \ --lhs-name ${system_llvm_release} \ - --rhs-name pgo \ + --rhs-name pgo-${yyyymmdd} \ ~/system/results.json vs ~/pgo/results.json > ~/results-system-vs-pgo.txt || true /root/test-suite/utils/compare.py \ --metric compile_time \ - --metric link_time \ --lhs-name ${system_llvm_release} \ - --rhs-name big-merge} \ + --rhs-name big-merge-${yyyymmdd} \ ~/system/results.json vs ~/big-merge/results.json > ~/results-system-vs-big-merge.txt || true /root/test-suite/utils/compare.py \ --metric compile_time \ - --metric link_time \ --lhs-name big-merge \ - --rhs-name pgo \ + --rhs-name pgo-${yyyymmdd} \ ~/big-merge/results.json vs ~/pgo/results.json > ~/results-big-merge-vs-pgo.txt || true bash From 7f4855ad014fbab8dc8a8d7cbe1fd0219475ebfb Mon Sep 17 00:00:00 2001 From: Konrad Kleine Date: Thu, 12 Sep 2024 08:44:29 +0000 Subject: [PATCH 9/9] Disable perf in container but leave it there for later --- scripts/perf/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/perf/entrypoint.sh b/scripts/perf/entrypoint.sh index bc271f9e..34aac745 100755 --- a/scripts/perf/entrypoint.sh +++ b/scripts/perf/entrypoint.sh @@ -11,7 +11,7 @@ function configure_build_run { -DCMAKE_CXX_COMPILER=/usr/bin/clang++ \ -DTEST_SUITE_BENCHMARKING_ONLY=ON \ -DTEST_SUITE_COLLECT_STATS=ON \ - -DTEST_SUITE_USE_PERF=ON \ + -DTEST_SUITE_USE_PERF=OFF \ -DTEST_SUITE_SUBDIRS=CTMark \ -DTEST_SUITE_RUN_BENCHMARKS=OFF \ -C~/test-suite/cmake/caches/O3.cmake \