Skip to content

Commit

Permalink
ci workflows support new split htslib. support new wheel. suport wind…
Browse files Browse the repository at this point in the history
…ows build
  • Loading branch information
graphenn authored and brentp committed Dec 19, 2023
1 parent d32bf86 commit d2a39e8
Show file tree
Hide file tree
Showing 13 changed files with 253 additions and 107 deletions.
116 changes: 96 additions & 20 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,24 @@ on: [push, pull_request]

jobs:
docker-build:
name: Docker Run Test on ${{ matrix.platform }}
runs-on: ubuntu-latest
name: Docker Run Test on ${{ matrix.platform }}-${{ matrix.python_tag_type }}
runs-on: ubuntu-22.04
strategy:
matrix:
platform:
# No need to test on amd64
- linux/amd64
- linux/arm64
python_tag_type:
- slim
- alpine
exclude:
# amd64 with glibc have full direct test
- platform: linux/amd64
python_tag_type: slim

# test alpine only on amd64
- platform: linux/arm64
python_tag_type: alpine

steps:
- uses: actions/checkout@v4
Expand All @@ -20,40 +31,44 @@ jobs:
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Docker Build and Push

- name: Docker Build
uses: docker/build-push-action@v5
with:
context: .
file: ./ci/Dockerfile.test
file: ./ci/Dockerfile.${{ matrix.python_tag_type }}.test
platforms: ${{ matrix.platform }}
tags: cyvcf2:test
tags: cyvcf2:${{ matrix.python_tag_type }}-test
push: false
load: true
build-args: |
PYTHON_VERSION=slim
PYTHON_VERSION=${{ matrix.python_tag_type }}
- name: Docker Run Tests
run: |
docker run --rm --platform ${{ matrix.platform }} cyvcf2:test pytest --cov cyvcf2 --cov-report term-missing
docker run --rm --platform ${{ matrix.platform }} cyvcf2:${{ matrix.python_tag_type }}-test pytest --cov cyvcf2 --cov-report term-missing
build:
name: Run tests on Python ${{ matrix.python-version }} ${{ matrix.os }}
runs-on: ${{ matrix.os }}-latest
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu, macos]
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
os: [ubuntu-22.04, macos-12]
python-version:
["pypy3.10", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
exclude:
# Run only the latest versions on macOS
- os: macos
# Run only the latest versions on macOS and windows
- os: macos-12
python-version: "pypy3.10"
- os: macos-12
python-version: "3.7"
- os: macos
- os: macos-12
python-version: "3.8"
- os: macos
- os: macos-12
python-version: "3.9"
- os: macos
- os: macos-12
python-version: "3.10"
- os: macos
- os: macos-12
python-version: "3.11"

steps:
Expand All @@ -65,12 +80,20 @@ jobs:
with:
python-version: ${{ matrix.python-version }}

- name: Set macOS env
if: runner.os == 'macOS'
run: |
# building options
echo "MACOSX_DEPLOYMENT_TARGET=10.9" >> "$GITHUB_ENV"
echo "ARCHFLAGS=-arch x86_64" >> "$GITHUB_ENV"
- name: Install Linux build prerequisites
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends libcurl4-openssl-dev zlib1g-dev libssl-dev liblzma-dev \
libbz2-dev libdeflate-dev
- name: Install macOS build prerequisites
if: runner.os == 'macOS'
run: |
Expand All @@ -81,17 +104,70 @@ jobs:
pip install -r requirements.txt
pip install pytest pytest-cov
CYVCF2_HTSLIB_CONFIGURE_OPTIONS="--enable-libcurl --enable-s3 --enable-lzma --enable-bz2 --with-libdeflate" \
CYTHONIZE=1 MACOSX_DEPLOYMENT_TARGET=10.9 python setup.py build_ext -i
CYTHONIZE=1 python setup.py build_ext -i
- name: Test
run: |
pytest --cov cyvcf2 --cov-report term-missing
windows_build:
name: Run tests on Python windows-2022 MSYS2 UCRT64
runs-on: windows-2022
defaults:
run:
shell: msys2 {0}

steps:
- uses: actions/checkout@v4
with:
submodules: recursive

- name: "Setup MSYS2"
uses: msys2/setup-msys2@v2
with:
msystem: UCRT64
path-type: inherit
install: >-
mingw-w64-ucrt-x86_64-gcc
mingw-w64-ucrt-x86_64-make
mingw-w64-ucrt-x86_64-libdeflate
mingw-w64-ucrt-x86_64-xz
mingw-w64-ucrt-x86_64-curl
mingw-w64-ucrt-x86_64-zlib
mingw-w64-ucrt-x86_64-bzip2
mingw-w64-ucrt-x86_64-tools-git
mingw-w64-ucrt-x86_64-python-pkgconfig
mingw-w64-ucrt-x86_64-pkg-config
mingw-w64-ucrt-x86_64-ninja
mingw-w64-ucrt-x86_64-python
mingw-w64-ucrt-x86_64-python-pip
make
automake
autoconf
git
- name: Install Windows build prerequisites
run: |
cd htslib
autoreconf -i
./configure --enable-libcurl --enable-s3 --enable-lzma --enable-bz2 --with-libdeflate
make
make install
- name: Install
run: |
pip install -r requirements.txt
pip install pytest pytest-cov
CYTHONIZE=1 python setup.py build_ext -i
- name: Test
run: |
pytest --cov cyvcf2 --cov-report term-missing
sdist:
runs-on: ${{ matrix.os }}-latest
runs-on: ubuntu-22.04
strategy:
matrix:
os: [ubuntu]
python-version: ["3.12"]

steps:
Expand Down
58 changes: 43 additions & 15 deletions .github/workflows/wheels.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,43 +18,71 @@ on:

jobs:
build_wheels:
name: Build wheels on ${{ matrix.os }}
runs-on: ${{ matrix.os }}-latest
name: Build wheels for ${{ matrix.python-version }}-${{ matrix.buildplat[1] }}
runs-on: ${{ matrix.buildplat[0] }}
strategy:
matrix:
os: [ubuntu, macos]
python-version: ["3.12"]
buildplat:
- [ubuntu-22.04, manylinux_x86_64]
- [ubuntu-22.04, musllinux_x86_64]
- [ubuntu-22.04, manylinux_aarch64]
- [ubuntu-22.04, musllinux_aarch64]
# macOS build arm64 on x86_64 with cross-compiler
- [macos-12, macosx_x86_64]
python-version: [pp310, cp37, cp38, cp39, cp310, cp311, cp312]
exclude:
# pp310, cp37, cp38 on musllinux is not support
- buildplat: [ubuntu-22.04, musllinux_x86_64]
python-version: cp37
- buildplat: [ubuntu-22.04, musllinux_x86_64]
python-version: cp38
- buildplat: [ubuntu-22.04, musllinux_x86_64]
python-version: pp310
- buildplat: [ubuntu-22.04, musllinux_aarch64]
python-version: cp37
- buildplat: [ubuntu-22.04, musllinux_aarch64]
python-version: cp38
- buildplat: [ubuntu-22.04, musllinux_aarch64]
python-version: pp310

steps:
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}

- name: Set up QEMU
if: runner.os == 'Linux'
uses: docker/setup-qemu-action@v3

- name: Build wheels for Linux
- name: Build wheels
uses: pypa/[email protected]
if: runner.os == 'Linux'
with:
package-dir: .
output-dir: wheelhouse
config-file: "{package}/pyproject.toml"
env:
CIBW_SKIP: "pp* *i686*"
# select
CIBW_BUILD: ${{ matrix.python-version }}-${{ matrix.buildplat[1] }}

# linux
CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
CIBW_MUSLLINUX_X86_64_IMAGE: musllinux_1_1
# manylinux2014 can't build on aarch64
CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
CIBW_MUSLLINUX_X86_64_IMAGE: musllinux_1_2
CIBW_MUSLLINUX_AARCH64_IMAGE: musllinux_1_2
CIBW_ARCHS_LINUX: auto64 aarch64
CIBW_BEFORE_BUILD_LINUX: "{project}/ci/linux-deps"
CIBW_TEST_COMMAND: "{project}/ci/test"
CIBW_REPAIR_WHEEL_COMMAND_LINUX: 'LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib64" && auditwheel repair -w {dest_dir} {wheel}'

# macos
CIBW_ARCHS_MACOS: auto64 arm64
CIBW_BEFORE_BUILD_MACOS: "{project}/ci/osx-deps"

# build
CIBW_ENVIRONMENT: >-
CYVCF2_HTSLIB_CONFIGURE_OPTIONS="--enable-libcurl --enable-s3 --enable-lzma --enable-bz2 --with-libdeflate"
CYTHONIZE=1
CIBW_TEST_COMMAND: "{project}/ci/test"

# Enable tmate debugging of manually-triggered workflows if the input option was provided
- name: Setup tmate session
Expand All @@ -67,7 +95,7 @@ jobs:

build_sdist:
name: Build source distribution
runs-on: ubuntu-latest
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
with:
Expand All @@ -90,7 +118,7 @@ jobs:

upload_pypi:
needs: [build_wheels, build_sdist]
runs-on: ubuntu-latest
runs-on: ubuntu-22.04
# upload to PyPI on every tag starting with 'v'
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
steps:
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,4 @@ _templates
setup-requires/*
.cache/v/cache/lastfailed
.idea
**/__pycache__
15 changes: 15 additions & 0 deletions ci/Dockerfile.alpine.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
ARG PYTHON_VERSION=alpine

FROM python:${PYTHON_VERSION}

WORKDIR /workspace

RUN apk add --no-cache build-base autoconf automake git xz-dev curl-dev libdeflate-dev bzip2-dev

COPY . .

RUN pip install -r requirements.txt && pip install pytest pytest-cov

# build cyvcf2
RUN CYVCF2_HTSLIB_CONFIGURE_OPTIONS="--enable-libcurl --enable-s3 --enable-lzma --enable-bz2 --with-libdeflate" \
CYTHONIZE=1 python setup.py build_ext -i
File renamed without changes.
17 changes: 8 additions & 9 deletions ci/linux-deps
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,15 @@ echo "manylinux image woking on $ID"

# Install cyvcf2 development files.
case "$ID" in
alpine)
apk add --no-cache xz-dev curl-dev
almalinux)
dnf install -y bzip2-devel xz-devel libcurl-devel openssl-devel openblas-devel epel-release

LIBDEFLATE_VERSION=1.19
curl -L -o libdeflate-v"$LIBDEFLATE_VERSION".tar.gz https://github.com/ebiggers/libdeflate/archive/refs/tags/v"$LIBDEFLATE_VERSION".tar.gz
tar xzf libdeflate-v"$LIBDEFLATE_VERSION".tar.gz
cd libdeflate-"$LIBDEFLATE_VERSION"
cmake -B build && cmake --build build
cd ./build/ && make install
cd ../..
# packages at epel
dnf install -y libdeflate-devel
;;

alpine)
apk add --no-cache xz-dev curl-dev libdeflate-dev
;;

centos)
Expand Down
19 changes: 0 additions & 19 deletions ci/osx-arm64-deps

This file was deleted.

40 changes: 20 additions & 20 deletions ci/osx-deps
Original file line number Diff line number Diff line change
@@ -1,26 +1,26 @@
#!/bin/bash

set -e
set -euo pipefail

git submodule init
git submodule update
git submodule update --init --recursive
export DYLD_LIBRARY_PATH=/usr/local/lib
# same with python
export MACOSX_DEPLOYMENT_TARGET=10.9

# configure fails with autoconf 2.71, so downgrade
# see https://github.com/asdf-vm/asdf-erlang/issues/195
brew install [email protected] && \
brew link --overwrite [email protected] && \
autoconf -V
brew install automake
brew unlink xz

curl -L -o libdeflate-v1.8.tar.gz https://github.com/ebiggers/libdeflate/archive/refs/tags/v1.8.tar.gz
tar xzf libdeflate-v1.8.tar.gz
cd libdeflate-1.8
make
make install
cd ..
# build liblzma and libdelfate for muti arch

cd htslib
autoheader
autoconf
./configure --enable-libcurl --enable-s3 --enable-lzma --enable-bz2
make
XZ_VERSION=5.4.5
curl -L -o xz-$XZ_VERSION.tar.gz "https://github.com/tukaani-project/xz/releases/download/v$XZ_VERSION/xz-$XZ_VERSION.tar.gz"
tar -xf xz-$XZ_VERSION.tar.gz
cd xz-$XZ_VERSION
cmake -B build && cmake --build build
cd ./build && make install

LIBDEFLATE_VERSION=1.19
curl -L -o libdeflate-v"$LIBDEFLATE_VERSION".tar.gz https://github.com/ebiggers/libdeflate/archive/refs/tags/v"$LIBDEFLATE_VERSION".tar.gz
tar xzf libdeflate-v"$LIBDEFLATE_VERSION".tar.gz
cd libdeflate-"$LIBDEFLATE_VERSION"
cmake -B build && cmake --build build
cd ./build && make install
Loading

0 comments on commit d2a39e8

Please sign in to comment.