Skip to content

Commit 2330fcc

Browse files
tskisnertskisner
committed
Extensive fixes to packaging:
- Updates to conda and wheel dependency installs - Documentation updates - Fix warnings and linking errors with clang on macos - Add github workflow for documentation updates - Fixes to use of MPI in unit tests and with zarr output - Use modern PyPI deployment tokens
1 parent 5c4df8c commit 2330fcc

31 files changed

+701
-240
lines changed

‎.github/workflows/deploy.yml

Lines changed: 47 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,10 @@ jobs:
4242
arch: x86_64
4343
python: 312
4444
builder: manylinux
45+
- os: ubuntu-latest
46+
arch: x86_64
47+
python: 313
48+
builder: manylinux
4549

4650
# MacOS x86_64. The macos-13 runner is the last
4751
# Intel-based runner version. At some point we'll
@@ -50,45 +54,49 @@ jobs:
5054
arch: x86_64
5155
python: 310
5256
builder: macosx
53-
deploy: 13.0
5457
- os: macos-13
5558
arch: x86_64
5659
python: 311
5760
builder: macosx
58-
deploy: 13.0
5961
- os: macos-13
6062
arch: x86_64
6163
python: 312
6264
builder: macosx
63-
deploy: 13.0
65+
- os: macos-13
66+
arch: x86_64
67+
python: 313
68+
builder: macosx
6469

6570
# MacOS arm64
6671
- os: macos-latest
6772
arch: arm64
6873
python: 310
6974
builder: macosx
70-
deploy: 14.0
7175
- os: macos-latest
7276
arch: arm64
7377
python: 311
7478
builder: macosx
75-
deploy: 14.0
7679
- os: macos-latest
7780
arch: arm64
7881
python: 312
7982
builder: macosx
80-
deploy: 14.0
83+
- os: macos-latest
84+
arch: arm64
85+
python: 313
86+
builder: macosx
8187
env:
8288
CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.builder }}_${{ matrix.arch }}
8389
CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.builder }}2014
8490
CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.builder }}2014
8591
CIBW_BUILD_VERBOSITY: 3
86-
CIBW_ENVIRONMENT_LINUX:
92+
CIBW_ENVIRONMENT_LINUX: >
93+
CC=gcc
8794
CIBW_ENVIRONMENT_MACOS: >
88-
MACOSX_DEPLOYMENT_TARGET=${{ matrix.deploy }}
95+
CC=clang
8996
CIBW_BEFORE_BUILD_LINUX: ./packaging/wheels/install_deps_linux.sh
9097
CIBW_BEFORE_BUILD_MACOS: ./packaging/wheels/install_deps_osx.sh
91-
CIBW_BEFORE_TEST: export OMP_NUM_THREADS=1
98+
CIBW_BEFORE_TEST: export OMP_NUM_THREADS=2
99+
CIBW_TEST_REQUIRES: h5py zarr
92100
CIBW_TEST_COMMAND: python3 -c 'import flacarray.tests; flacarray.tests.run()'
93101
steps:
94102
- name: Checkout
@@ -97,16 +105,41 @@ jobs:
97105
- uses: actions/setup-python@v5
98106
name: Install Python
99107
with:
100-
python-version: '3.10'
108+
python-version: '3.11'
101109

102110
- name: Install cibuildwheel
103111
run: |
104-
python -m pip install twine cibuildwheel==2.20.0
112+
python -m pip install twine cibuildwheel==2.22.0
105113
106114
- name: Build wheel
107115
run: |
108116
python -m cibuildwheel --output-dir wheelhouse
109117
110-
- name: Upload to PyPI
111-
run: |
112-
python -m twine upload wheelhouse/flacarray*cp${{ matrix.python }}-${{ matrix.builder }}*${{ matrix.arch }}*.whl
118+
build_sdist:
119+
name: Build source distribution
120+
runs-on: ubuntu-latest
121+
steps:
122+
- uses: actions/checkout@v4
123+
124+
- name: Build sdist
125+
run: pipx run build --sdist
126+
127+
- uses: actions/upload-artifact@v4
128+
with:
129+
name: flacarray-sdist
130+
path: dist/*.tar.gz
131+
132+
upload_pypi:
133+
needs: [build_wheels, build_sdist]
134+
runs-on: ubuntu-latest
135+
permissions:
136+
id-token: write
137+
steps:
138+
- uses: actions/download-artifact@v4
139+
with:
140+
# Unpacks all artifacts into dist/
141+
pattern: flacarray-*
142+
path: dist
143+
merge-multiple: true
144+
145+
- uses: pypa/gh-action-pypi-publish@release/v1

‎.github/workflows/docs.yml

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: Deploy Documentation
2+
3+
on:
4+
workflow_dispatch:
5+
push:
6+
tags:
7+
- '*'
8+
9+
concurrency:
10+
group: ${{ github.workflow }}-${{ github.ref }}
11+
cancel-in-progress: true
12+
13+
jobs:
14+
update_docs:
15+
name: Build and Deploy Documentation
16+
runs-on: ubuntu-latest
17+
defaults:
18+
run:
19+
shell: bash -l {0}
20+
steps:
21+
- name: Checkout
22+
uses: actions/checkout@v4
23+
24+
- name: Setup Python
25+
uses: actions/setup-python@v5
26+
with:
27+
python-version: 3.x
28+
29+
- name: Create Cache ID
30+
run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
31+
32+
- name: Setup Cache
33+
uses: actions/cache@v4
34+
with:
35+
key: mkdocs-material-${{ env.cache_id }}
36+
path: .cache
37+
restore-keys: |
38+
mkdocs-material-
39+
40+
- name: Install Dependencies
41+
run: |
42+
pip install mkdocs mkdocstrings mkdocstrings-python mkdocs-material mkdocs-material-extensions mkdocs-jupyter
43+
git config user.name 'github-actions[bot]'
44+
git config user.email 'github-actions[bot]@users.noreply.github.com'
45+
46+
- name: Deploy Docs
47+
run: |
48+
cd docs
49+
mkdocs gh-deploy --force

‎.github/workflows/test.yml

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -33,27 +33,27 @@ jobs:
3333
arch: Linux-x86_64
3434
ompdisable: 0
3535
- os: ubuntu-latest
36-
python: "3.10"
36+
python: "3.11"
3737
arch: Linux-x86_64
3838
ompdisable: 0
3939
- os: ubuntu-latest
40-
python: "3.12"
40+
python: "3.13"
4141
arch: Linux-x86_64
4242
ompdisable: 0
4343
- os: macos-latest
4444
python: "3.10"
4545
arch: MacOSX-x86_64
4646
ompdisable: 1
4747
- os: macos-latest
48-
python: "3.12"
48+
python: "3.13"
4949
arch: MacOSX-x86_64
5050
ompdisable: 1
5151
- os: macos-latest
5252
python: "3.10"
5353
arch: MacOSX-arm64
5454
ompdisable: 1
5555
- os: macos-latest
56-
python: "3.12"
56+
python: "3.13"
5757
arch: MacOSX-arm64
5858
ompdisable: 1
5959
steps:
@@ -82,25 +82,32 @@ jobs:
8282
- name: Install Dependencies
8383
run: |
8484
source ~/conda/etc/profile.d/conda.sh \
85-
&& conda activate base \
86-
&& conda install --file packaging/conda_build_requirements.txt
85+
&& conda create --yes -n test python==3.12 \
86+
&& conda activate test \
87+
&& conda install --yes --file packaging/conda_build_requirements.txt
8788
8889
- name: Install
8990
run: |
9091
source ~/conda/etc/profile.d/conda.sh \
91-
&& conda activate base \
92+
&& conda activate test \
9293
&& pip install .
9394
9495
- name: Run Serial Tests
9596
run: |
9697
source ~/conda/etc/profile.d/conda.sh \
97-
&& conda activate base \
98+
&& conda activate test \
9899
&& export OMP_NUM_THREADS=2 \
99-
&& python3 -c 'import flacarray.tests; flacarray.tests.run()'
100+
&& mkdir -p test \
101+
&& pushd test >/dev/null 2>&1 \
102+
&& python3 -c 'import flacarray.tests; flacarray.tests.run()' \
103+
&& popd >/dev/null 2>&1
100104
101105
- name: Run MPI Tests
102106
run: |
103107
source ~/conda/etc/profile.d/conda.sh \
104-
&& conda activate base \
108+
&& conda activate test \
105109
&& export OMP_NUM_THREADS=1 \
106-
&& mpirun -np 2 python -c 'import flacarray.tests; flacarray.tests.run()'
110+
&& mkdir -p test \
111+
&& pushd test >/dev/null 2>&1 \
112+
&& mpirun -np 2 python -m mpi4py -c 'import flacarray.tests; flacarray.tests.run()' \
113+
&& popd >/dev/null 2>&1

‎.github/workflows/wheels.yml

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ jobs:
3434
arch: x86_64
3535
python: 312
3636
builder: manylinux
37+
- os: ubuntu-latest
38+
arch: x86_64
39+
python: 313
40+
builder: manylinux
3741

3842
# MacOS x86_64. The macos-13 runner is the last
3943
# Intel-based runner version. At some point we'll
@@ -42,61 +46,62 @@ jobs:
4246
arch: x86_64
4347
python: 310
4448
builder: macosx
45-
deploy: 13.0
4649
- os: macos-13
4750
arch: x86_64
4851
python: 311
4952
builder: macosx
50-
deploy: 13.0
5153
- os: macos-13
5254
arch: x86_64
5355
python: 312
5456
builder: macosx
55-
deploy: 13.0
57+
- os: macos-13
58+
arch: x86_64
59+
python: 313
60+
builder: macosx
5661

5762
# MacOS arm64
5863
- os: macos-latest
5964
arch: arm64
6065
python: 310
6166
builder: macosx
62-
deploy: 14.0
6367
- os: macos-latest
6468
arch: arm64
6569
python: 311
6670
builder: macosx
67-
deploy: 14.0
6871
- os: macos-latest
6972
arch: arm64
7073
python: 312
7174
builder: macosx
72-
deploy: 14.0
75+
- os: macos-latest
76+
arch: arm64
77+
python: 313
78+
builder: macosx
7379
env:
7480
CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.builder }}_${{ matrix.arch }}
7581
CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.builder }}2014
7682
CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.builder }}2014
7783
CIBW_BUILD_VERBOSITY: 3
78-
CIBW_ENVIRONMENT_LINUX:
84+
CIBW_ENVIRONMENT_LINUX: >
85+
CC=gcc
7986
CIBW_ENVIRONMENT_MACOS: >
80-
MACOSX_DEPLOYMENT_TARGET=${{ matrix.deploy }}
87+
CC=clang
8188
CIBW_BEFORE_BUILD_LINUX: ./packaging/wheels/install_deps_linux.sh
8289
CIBW_BEFORE_BUILD_MACOS: ./packaging/wheels/install_deps_osx.sh
90+
CIBW_TEST_REQUIRES: h5py zarr
8391
CIBW_BEFORE_TEST: export OMP_NUM_THREADS=2
84-
CIBW_TEST_COMMAND_LINUX: >
85-
python -c 'import flacarray.tests; flacarray.tests.run()'
86-
CIBW_TEST_COMMAND_MACOS: >
87-
python -c 'import flacarray.tests; flacarray.tests.run()'
92+
CIBW_TEST_COMMAND: python3 -c 'import flacarray.tests; flacarray.tests.run()'
8893
steps:
8994
- name: Checkout
9095
uses: actions/checkout@v4
9196

9297
- uses: actions/setup-python@v5
9398
name: Install Python
9499
with:
95-
python-version: '3.10'
100+
python-version: '3.11'
96101

97102
- name: Install cibuildwheel
98103
run: |
99-
python -m pip install cibuildwheel==2.20.0
104+
python -m pip install cibuildwheel==2.22.0
100105
101106
- name: Build wheel
102107
run: |

‎README.md

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,11 @@
11
# FLACArray
22

3-
FLAC Compression of Numpy Arrays.
3+
![Build Status](https://github.com/hpc4cmb/flacarray/workflows/Run%20Test%20Suite/badge.svg?branch=main)
4+
5+
Tools for FLAC compression of numpy arrays in memory and support for several
6+
on-disk formats. This is intended for use with raw numerical data rather than
7+
audio formats.
8+
9+
## Documentation
10+
11+
[See the full documentation here](https://hpc4cmb.github.io/flacarray).

0 commit comments

Comments
 (0)