benchmark.yml

name: Benchmark
on:
  pull_request:
    paths:
      - '.github/workflows/benchmark.yml'
      - 'src/**'
      - 'tests/**'
  workflow_dispatch:
    inputs:
      base_commit:
        description: The base commit to compare against


permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

env:
  CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse

jobs:
  benchmark:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        timeout-minutes: 3
        with:
          persist-credentials: false
          path: "cryptography-pr"
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        timeout-minutes: 3
        with:
          persist-credentials: false
          repository: "pyca/cryptography"
          path: "cryptography-base"
          ref: "${{ github.event.inputs.base_commit || github.base_ref }}"
      - name: Clone test vectors
        timeout-minutes: 2
        uses: ./cryptography-base/.github/actions/fetch-vectors

      - name: Setup python
        id: setup-python
        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
        with:
          python-version: "3.11"

      - name: Create virtualenv (base)
        run: |
          python -m venv .venv-base
          .venv-base/bin/pip install -v -c ./cryptography-base/ci-constraints-requirements.txt "./cryptography-base[test]" ./cryptography-base/vectors/
      - name: Create virtualenv (PR)
        run: |
          python -m venv .venv-pr
          .venv-pr/bin/pip install -v -c ./cryptography-pr/ci-constraints-requirements.txt "./cryptography-pr[test]" ./cryptography-pr/vectors/

      - name: Run benchmarks (base)
        run: .venv-base/bin/pytest --benchmark-enable --benchmark-only ./cryptography-pr/tests/bench/ --benchmark-json=bench-base.json --x509-limbo-root=x509-limbo/
      - name: Run benchmarks (PR)
        run: .venv-pr/bin/pytest --benchmark-enable --benchmark-only ./cryptography-pr/tests/bench/ --benchmark-json=bench-pr.json --x509-limbo-root=x509-limbo/

      - name: Compare results
        run: python ./cryptography-pr/.github/compare_benchmarks.py bench-base.json bench-pr.json | tee -a $GITHUB_STEP_SUMMARY