Run benchmarks in the correct context and compare main vs PR

.github/workflows/benchmarks_comment.yml

@@ -0,0 +1,91 @@
+name: Add Benchmark Results to PR
+on:
+  workflow_run:
+    workflows: [ Run and Cache Benchmarks ]
+    types: [ completed ]
+
+jobs:
+  comment:
+    if: github.event.workflow_run.conclusion == 'success'
+    runs-on: ubuntu-latest
+    env:
+      BENCHMARK_RESULTS: benchmark_results
+      PR_EVENT: event.json
+    steps:
+      - name: Download Benchmark Results
+        uses: actions/github-script@v6
+        with:
+          script: |
+            async function downloadArtifact(artifactName) {
+              let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: context.payload.workflow_run.id,
+              });
+              let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+                return artifact.name == artifactName
+              })[0];
+              if (!matchArtifact) {
+                core.setFailed(`Failed to find artifact: ${artifactName}`);
+              }
+              let download = await github.rest.actions.downloadArtifact({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                artifact_id: matchArtifact.id,
+                archive_format: 'zip',
+              });
+              let fs = require('fs');
+              fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/${artifactName}.zip`, Buffer.from(download.data));
+            }
+            await downloadArtifact(process.env.BENCHMARK_RESULTS);
+            await downloadArtifact(process.env.PR_EVENT);
+      - name: Unzip Benchmark Results
+        run: |
+          unzip $BENCHMARK_RESULTS.zip
+          unzip $PR_EVENT.zip
+      - name: Export PR Event Data
+        uses: actions/github-script@v6
+        with:
+          script: |
+            let fs = require('fs');
+            let prEvent = JSON.parse(fs.readFileSync(process.env.PR_EVENT, {encoding: 'utf8'}));
+            core.exportVariable("PR_NUMBER", prEvent.number);
+
+      - name: Find Comment
+        uses: peter-evans/find-comment@v3
+        with:
+          issue-number: ${{ env.PR_NUMBER }}
+          comment-author: 'github-actions[bot]'
+          body-includes: Benchmark Results - Solvers
+
+      - name: Create or update comment
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const issue_number = ${{ env.PR_NUMBER }};
+            const comment_id = '${{ steps.fc.outputs.comment-id }}';
+            let results_main = 'Benchmarks did not run successfully on main.';
+            let results_pr = 'Benchmarks did not run successfully on PR.';
+            if (fs.existsSync('results_main.txt')) {
+              results_main = fs.readFileSync('results_main.txt', 'utf8');
+            }
+            if (fs.existsSync('results_pr.txt')) {
+              results_pr = fs.readFileSync('results_pr.txt', 'utf8');
+            }
+            const body = `Benchmark Results - Solvers\n\n<details><summary>Main</summary>\n\n\`\`\`\n${results_main}\n\`\`\`\n</details>\n<details><summary>PR</summary>\n\n\`\`\`\n${results_pr}\n\`\`\`\n</details>`;
+            if (comment_id) {
+              github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: comment_id,
+                body: body,
+              });
+            } else {
+              github.rest.issues.createComment({
+                issue_number: issue_number,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: body,
+              });
+            }

.github/workflows/benchmarks_run.yml

@@ -0,0 +1,40 @@
+name: Run and Cache Benchmarks
+
+on:
+  pull_request:
+    types: [ opened, reopened, edited, synchronize ]
+
+jobs:
+  benchmark:
+    name: Run benchmarks
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout main branch
+        uses: actions/checkout@v2
+        with:
+          ref: main
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Run benchmarks on main - solvers
+        run: cargo bench -- solvers --exact | tail -n +6  > results_main.txt
+
+      - name: Checkout PR branch
+        uses: actions/checkout@v2
+
+      - name: Run benchmarks on PR branch
+        run: cargo bench -- solvers --exact | tail -n +6 > results_pr.txt
+
+      - name: Upload Benchmark Results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark_results
+          path: ./results_*.txt
+          if-no-files-found: error
+
+      - name: Upload GitHub Pull Request Event
+        uses: actions/upload-artifact@v4
+        with:
+          name: event.json
+          path: ${{ github.event_path }}