Skip to content

Commit f0d0c88

Browse files
davidhewittdavidhewitt
committed
fix linux PGO wheel build (#1557)
1 parent 3165513 commit f0d0c88

File tree

5 files changed

+226
-64
lines changed

5 files changed

+226
-64
lines changed

‎.github/actions/build-pgo-wheel/action.yml‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
name: Build PGO wheel
2+
description: Builds a PGO-optimized wheel
3+
inputs:
4+
interpreter:
5+
description: 'Interpreter to build the wheel for'
6+
required: true
7+
rust-toolchain:
8+
description: 'Rust toolchain to use'
9+
required: true
10+
outputs:
11+
wheel:
12+
description: 'Path to the built wheel'
13+
value: ${{ steps.find_wheel.outputs.path }}
14+
runs:
15+
using: "composite"
16+
steps:
17+
- name: prepare self schema
18+
shell: bash
19+
# generate up front so that we don't have to do this inside the docker container
20+
run: uv run python generate_self_schema.py
21+
22+
- name: prepare profiling directory
23+
shell: bash
24+
# making this ahead of the compile ensures that the local user can write to this
25+
# directory; the maturin action (on linux) runs in docker so would create as root
26+
run: mkdir -p ${{ github.workspace }}/profdata
27+
28+
- name: build initial wheel
29+
uses: PyO3/maturin-action@v1
30+
with:
31+
manylinux: auto
32+
args: >
33+
--release
34+
--out pgo-wheel
35+
--interpreter ${{ inputs.interpreter }}
36+
rust-toolchain: ${{ inputs.rust-toolchain }}
37+
docker-options: -e CI
38+
env:
39+
RUSTFLAGS: '-Cprofile-generate=${{ github.workspace }}/profdata'
40+
41+
- name: detect rust host
42+
run: echo RUST_HOST=$(rustc -Vv | grep host | cut -d ' ' -f 2) >> "$GITHUB_ENV"
43+
shell: bash
44+
45+
- name: generate pgo data
46+
run: |
47+
uv sync --group testing
48+
uv pip install pydantic-core --no-index --no-deps --find-links pgo-wheel --force-reinstall
49+
uv run pytest tests/benchmarks
50+
RUST_HOST=$(rustc -Vv | grep host | cut -d ' ' -f 2)
51+
rustup run ${{ inputs.rust-toolchain }} bash -c 'echo LLVM_PROFDATA=$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/$RUST_HOST/bin/llvm-profdata >> "$GITHUB_ENV"'
52+
shell: bash
53+
54+
- name: merge pgo data
55+
run: ${{ env.LLVM_PROFDATA }} merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata
56+
shell: bash
57+
58+
- name: build pgo-optimized wheel
59+
uses: PyO3/maturin-action@v1
60+
with:
61+
manylinux: auto
62+
args: >
63+
--release
64+
--out dist
65+
--interpreter ${{ inputs.interpreter }}
66+
rust-toolchain: ${{inputs.rust-toolchain}}
67+
docker-options: -e CI
68+
env:
69+
RUSTFLAGS: '-Cprofile-use=${{ github.workspace }}/merged.profdata'
70+
71+
- name: find built wheel
72+
id: find_wheel
73+
run: echo "path=$(ls dist/*.whl)" | tee -a "$GITHUB_OUTPUT"
74+
shell: bash

‎.github/workflows/ci.yml‎

Lines changed: 4 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -556,50 +556,12 @@ jobs:
556556
# FIXME: Unpin when Python 3.8 support is dropped. (3.9 requires Windows 10)
557557
toolchain: ${{ (matrix.os == 'windows' && '1.77') || 'stable' }}
558558

559-
- run: pip install -U 'ruff==0.5.0' typing_extensions
560-
561-
# generate self-schema now, so we don't have to do so inside docker in maturin build
562-
- run: python generate_self_schema.py
563-
564-
- name: build initial wheel
565-
uses: PyO3/maturin-action@v1
559+
- name: Build PGO wheel
560+
id: pgo-wheel
561+
uses: ./.github/actions/build-pgo-wheel
566562
with:
567-
manylinux: auto
568-
args: >
569-
--release
570-
--out pgo-wheel
571-
--interpreter ${{ matrix.interpreter }}
563+
interpreter: ${{ env.UV_PYTHON }}
572564
rust-toolchain: ${{ steps.rust-toolchain.outputs.name }}
573-
docker-options: -e CI
574-
env:
575-
RUSTFLAGS: '-Cprofile-generate=${{ github.workspace }}/profdata'
576-
577-
- name: detect rust host
578-
run: echo RUST_HOST=$(rustc -Vv | grep host | cut -d ' ' -f 2) >> "$GITHUB_ENV"
579-
shell: bash
580-
581-
- name: generate pgo data
582-
run: |
583-
uv sync --group testing
584-
uv pip install pydantic-core --no-index --no-deps --find-links pgo-wheel --force-reinstall
585-
uv run pytest tests/benchmarks
586-
rustup run ${{ steps.rust-toolchain.outputs.name }} bash -c 'echo LLVM_PROFDATA=$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/${{ env.RUST_HOST }}/bin/llvm-profdata >> "$GITHUB_ENV"'
587-
588-
- name: merge pgo data
589-
run: ${{ env.LLVM_PROFDATA }} merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata
590-
591-
- name: build pgo-optimized wheel
592-
uses: PyO3/maturin-action@v1
593-
with:
594-
manylinux: auto
595-
args: >
596-
--release
597-
--out dist
598-
--interpreter ${{ matrix.interpreter }}
599-
rust-toolchain: ${{steps.rust-toolchain.outputs.name}}
600-
docker-options: -e CI
601-
env:
602-
RUSTFLAGS: '-Cprofile-use=${{ github.workspace }}/merged.profdata'
603565

604566
- run: ${{ matrix.ls || 'ls -lh' }} dist/
605567

‎.github/workflows/codspeed.yml‎

Lines changed: 15 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -30,40 +30,36 @@ jobs:
3030
with:
3131
enable-cache: true
3232

33-
- name: install deps
33+
- name: Install deps
3434
run: |
3535
uv sync --group testing
3636
uv pip uninstall pytest-speed
3737
uv pip install pytest-benchmark==4.0.0 pytest-codspeed
3838
39-
- name: install rust stable
39+
- name: Install rust stable
4040
id: rust-toolchain
4141
uses: dtolnay/rust-toolchain@stable
4242
with:
4343
components: llvm-tools
4444

45-
- name: cache rust
45+
- name: Cache rust
4646
uses: Swatinem/rust-cache@v2
4747

48-
- name: Compile pydantic-core for profiling
49-
run: make build-profiling
48+
- name: Build PGO wheel
49+
id: pgo-wheel
50+
uses: ./.github/actions/build-pgo-wheel
51+
with:
52+
interpreter: ${{ env.UV_PYTHON }}
53+
rust-toolchain: ${{ steps.rust-toolchain.outputs.name }}
5054
env:
51-
CONST_RANDOM_SEED: 0 # Fix the compile time RNG seed
52-
RUSTFLAGS: "-Cprofile-generate=${{ github.workspace }}/profdata"
53-
54-
- name: Gather pgo data
55-
run: uv run pytest tests/benchmarks
55+
# make sure profiling information is present
56+
CARGO_PROFILE_RELEASE_DEBUG: "line-tables-only"
57+
CARGO_PROFILE_RELEASE_STRIP: false
5658

57-
- name: Prepare merged pgo data
58-
run: rustup run stable bash -c '$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/x86_64-unknown-linux-gnu/bin/llvm-profdata merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata'
59-
60-
- name: Compile pydantic-core for benchmarking
61-
run: make build-profiling
62-
env:
63-
CONST_RANDOM_SEED: 0 # Fix the compile time RNG seed
64-
RUSTFLAGS: "-Cprofile-use=${{ github.workspace }}/merged.profdata"
59+
- name: Install PGO wheel
60+
run: uv pip install ${{ steps.pgo-wheel.outputs.wheel }} --force-reinstall
6561

6662
- name: Run CodSpeed benchmarks
6763
uses: CodSpeedHQ/action@v3
6864
with:
69-
run: uv run pytest tests/benchmarks/ --codspeed
65+
run: uv run --group=codspeed pytest tests/benchmarks/ --codspeed

‎pyproject.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,10 @@ wasm = [
8282
'maturin>=1,<2',
8383
'ruff',
8484
]
85+
codspeed = [
86+
# codspeed is only run on CI, with latest version of CPython
87+
'pytest-codspeed; python_version == "3.13" and implementation_name == "cpython"',
88+
]
8589

8690
all = [
8791
{ include-group = 'testing' },

0 commit comments

Comments
 (0)