Skip to content

Commit 9ad1425

Browse files
davidhewittdavidhewitt
authored
build with PGO on x86_64 ubuntu and windows (#678)
1 parent 024efa2 commit 9ad1425

File tree

2 files changed

+135
-14
lines changed

2 files changed

+135
-14
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 115 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -287,10 +287,25 @@ jobs:
287287
jobs: ${{ toJSON(needs) }}
288288
allowed-failures: coverage
289289

290+
build-sdist:
291+
name: build sdist
292+
runs-on: ubuntu-latest
293+
steps:
294+
- uses: actions/checkout@v3
295+
- uses: PyO3/maturin-action@v1
296+
with:
297+
command: sdist
298+
args: --out dist
299+
rust-toolchain: stable
300+
- uses: actions/upload-artifact@v3
301+
with:
302+
name: pypi_files
303+
path: dist
304+
290305
build:
291306
name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }})
292307
# only run on push to main and on release
293-
if: "success() && (startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build'))"
308+
if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build')
294309
strategy:
295310
fail-fast: false
296311
matrix:
@@ -345,6 +360,13 @@ jobs:
345360
container: messense/manylinux_2_24-cross:s390x
346361
interpreter: 3.7 3.8 3.9 3.10 3.11 3.12
347362
exclude:
363+
# Optimized PGO builds for x86_64 manylinux and windows follow a different matrix,
364+
# maybe in future maturin-action can support this automatically
365+
- os: ubuntu
366+
target: x86_64
367+
manylinux: auto
368+
- os: windows
369+
target: x86_64
348370
# Windows on arm64 only supports Python 3.11+
349371
- os: windows
350372
target: aarch64
@@ -364,14 +386,6 @@ jobs:
364386
# generate self-schema now, so we don't have to do so inside docker in maturin build
365387
- run: python generate_self_schema.py
366388

367-
- name: build sdist
368-
if: ${{ matrix.os == 'ubuntu' && matrix.target == 'x86_64' && matrix.manylinux == 'auto' }}
369-
uses: PyO3/maturin-action@v1
370-
with:
371-
command: sdist
372-
args: --out dist
373-
rust-toolchain: stable
374-
375389
- name: build wheels
376390
uses: PyO3/maturin-action@v1
377391
with:
@@ -391,8 +405,96 @@ jobs:
391405
name: pypi_files
392406
path: dist
393407

408+
build-pgo:
409+
name: build pgo-optimized on ${{ matrix.platform || matrix.os }} (${{ matrix.interpreter}} - ${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }})
410+
# only run on push to main and on release
411+
if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build')
412+
strategy:
413+
fail-fast: false
414+
matrix:
415+
os: [ubuntu, windows]
416+
target: [x86_64]
417+
manylinux: [auto]
418+
interpreter: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12-dev", "pypy3.7", "pypy3.8", "pypy3.9"]
419+
include:
420+
- os: ubuntu
421+
platform: linux
422+
- os: windows
423+
ls: dir
424+
- interpreter: 3.12-dev
425+
maturin-interpreter: "3.12"
426+
427+
runs-on: ${{ matrix.os }}-latest
428+
steps:
429+
- uses: actions/checkout@v3
430+
431+
- name: set up python
432+
uses: actions/setup-python@v4
433+
with:
434+
python-version: ${{ matrix.interpreter }}
435+
architecture: ${{ matrix.python-architecture || 'x64' }}
436+
437+
- name: install rust stable
438+
id: rust-toolchain
439+
uses: dtolnay/rust-toolchain@stable
440+
with:
441+
components: llvm-tools
442+
443+
- run: pip install -U 'black>=22.3.0,<23' typing_extensions
444+
445+
# generate self-schema now, so we don't have to do so inside docker in maturin build
446+
- run: python generate_self_schema.py
447+
448+
- name: build initial wheel
449+
uses: PyO3/maturin-action@v1
450+
with:
451+
target: ${{ matrix.target }}
452+
manylinux: ${{ matrix.manylinux || 'auto' }}
453+
args: >
454+
--release
455+
--out pgo-wheel
456+
--interpreter ${{ matrix.maturin-interpreter || matrix.interpreter }}
457+
-- -Cprofile-generate=${{ github.workspace }}/profdata
458+
rust-toolchain: stable
459+
docker-options: -e CI
460+
461+
- name: detect rust host
462+
run: echo RUST_HOST=$(rustc -Vv | grep host | cut -d ' ' -f 2) >> "$GITHUB_ENV"
463+
shell: bash
464+
465+
- name: generate pgo data
466+
run: |
467+
pip install -U pip
468+
pip install -r tests/requirements.txt
469+
pip install pydantic-core --no-index --no-deps --find-links pgo-wheel --force-reinstall
470+
pytest tests/benchmarks
471+
rustup run stable bash -c 'echo LLVM_PROFDATA=$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/${{ env.RUST_HOST }}/bin/llvm-profdata >> "$GITHUB_ENV"'
472+
473+
- name: merge pgo data
474+
run: ${{ env.LLVM_PROFDATA }} merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata
475+
476+
- name: build pgo-optimized wheel
477+
uses: PyO3/maturin-action@v1
478+
with:
479+
target: ${{ matrix.target }}
480+
manylinux: ${{ matrix.manylinux || 'auto' }}
481+
args: >
482+
--release
483+
--out dist
484+
--interpreter ${{ matrix.maturin-interpreter || matrix.interpreter }}
485+
-- -Cprofile-use=${{ github.workspace }}/merged.profdata
486+
rust-toolchain: stable
487+
docker-options: -e CI
488+
489+
- run: ${{ matrix.ls || 'ls -lh' }} dist/
490+
491+
- uses: actions/upload-artifact@v3
492+
with:
493+
name: pypi_files
494+
path: dist
495+
394496
inspect-pypi-assets:
395-
needs: [build]
497+
needs: [build, build-sdist, build-pgo]
396498
runs-on: ubuntu-latest
397499

398500
steps:
@@ -474,7 +576,7 @@ jobs:
474576
475577
test-builds-os:
476578
name: test build on ${{ matrix.os }}
477-
needs: [build]
579+
needs: [build, build-pgo]
478580

479581
strategy:
480582
fail-fast: false
@@ -502,8 +604,8 @@ jobs:
502604
- run: pytest --ignore=tests/test_docstrings.py
503605

504606
release:
505-
needs: [test-builds-arch, test-builds-os, check]
506-
if: "success() && startsWith(github.ref, 'refs/tags/')"
607+
needs: [test-builds-arch, test-builds-os, build-sdist, check]
608+
if: success() && startsWith(github.ref, 'refs/tags/')
507609
runs-on: ubuntu-latest
508610

509611
steps:

‎.github/workflows/codspeed.yml‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,18 +41,37 @@ jobs:
4141
if: steps.cache-py.outputs.cache-hit != 'true'
4242

4343
- name: install rust stable
44+
id: rust-toolchain
4445
uses: dtolnay/rust-toolchain@stable
46+
with:
47+
components: llvm-tools
4548

4649
- name: cache rust
4750
uses: Swatinem/rust-cache@v2
4851

49-
- name: Install pydantic-core
52+
- name: Compile pydantic-core for profiling
53+
# --no-default-features to avoid using mimalloc
54+
run: |
55+
pip install -e . --config-settings=build-args='--no-default-features --verbose' -v
56+
python -c 'import pydantic_core; assert pydantic_core._pydantic_core.__pydantic_core_default_allocator__'
57+
env:
58+
CONST_RANDOM_SEED: 0 # Fix the compile time RNG seed
59+
RUSTFLAGS: "-Cprofile-generate=${{ github.workspace }}/profdata"
60+
61+
- name: Gather pgo data
62+
run: pytest tests/benchmarks
63+
64+
- name: Prepare merged pgo data
65+
run: rustup run stable bash -c '$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/x86_64-unknown-linux-gnu/bin/llvm-profdata merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata'
66+
67+
- name: Compile pydantic-core for benchmarking
5068
# --no-default-features to avoid using mimalloc
5169
run: |
5270
pip install -e . --config-settings=build-args='--no-default-features --verbose' -v
5371
python -c 'import pydantic_core; assert pydantic_core._pydantic_core.__pydantic_core_default_allocator__'
5472
env:
5573
CONST_RANDOM_SEED: 0 # Fix the compile time RNG seed
74+
RUSTFLAGS: "-Cprofile-use=${{ github.workspace }}/merged.profdata"
5675

5776
- name: Run CodSpeed benchmarks
5877
uses: CodSpeedHQ/action@v1

0 commit comments

Comments
 (0)