diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b1f964b..fb7edfb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,13 +4,24 @@ on: [push, pull_request] jobs: docker-build: - name: Docker Run Test on ${{ matrix.platform }} - runs-on: ubuntu-latest + name: Docker Run Test on ${{ matrix.platform }}-${{ matrix.python_tag_type }} + runs-on: ubuntu-22.04 strategy: matrix: platform: - # No need to test on amd64 + - linux/amd64 - linux/arm64 + python_tag_type: + - slim + - alpine + exclude: + # amd64 with glibc have full direct test + - platform: linux/amd64 + python_tag_type: slim + + # test alpine only on amd64 + - platform: linux/arm64 + python_tag_type: alpine steps: - uses: actions/checkout@v4 @@ -20,40 +31,44 @@ jobs: uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Docker Build and Push + + - name: Docker Build uses: docker/build-push-action@v5 with: context: . - file: ./ci/Dockerfile.test + file: ./ci/Dockerfile.${{ matrix.python_tag_type }}.test platforms: ${{ matrix.platform }} - tags: cyvcf2:test + tags: cyvcf2:${{ matrix.python_tag_type }}-test push: false load: true build-args: | - PYTHON_VERSION=slim + PYTHON_VERSION=${{ matrix.python_tag_type }} - name: Docker Run Tests run: | - docker run --rm --platform ${{ matrix.platform }} cyvcf2:test pytest --cov cyvcf2 --cov-report term-missing + docker run --rm --platform ${{ matrix.platform }} cyvcf2:${{ matrix.python_tag_type }}-test pytest --cov cyvcf2 --cov-report term-missing build: name: Run tests on Python ${{ matrix.python-version }} ${{ matrix.os }} - runs-on: ${{ matrix.os }}-latest + runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu, macos] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + os: [ubuntu-22.04, macos-12] + python-version: + ["pypy3.10", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] exclude: - # Run only the latest versions on macOS - - os: macos + # Run only the latest versions on macOS and windows + - os: macos-12 + python-version: "pypy3.10" + - os: macos-12 python-version: "3.7" - - os: macos + - os: macos-12 python-version: "3.8" - - os: macos + - os: macos-12 python-version: "3.9" - - os: macos + - os: macos-12 python-version: "3.10" - - os: macos + - os: macos-12 python-version: "3.11" steps: @@ -65,12 +80,20 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Set macOS env + if: runner.os == 'macOS' + run: | + # building options + echo "MACOSX_DEPLOYMENT_TARGET=10.9" >> "$GITHUB_ENV" + echo "ARCHFLAGS=-arch x86_64" >> "$GITHUB_ENV" + - name: Install Linux build prerequisites if: runner.os == 'Linux' run: | sudo apt-get update sudo apt-get install -y --no-install-recommends libcurl4-openssl-dev zlib1g-dev libssl-dev liblzma-dev \ libbz2-dev libdeflate-dev + - name: Install macOS build prerequisites if: runner.os == 'macOS' run: | @@ -81,17 +104,70 @@ jobs: pip install -r requirements.txt pip install pytest pytest-cov CYVCF2_HTSLIB_CONFIGURE_OPTIONS="--enable-libcurl --enable-s3 --enable-lzma --enable-bz2 --with-libdeflate" \ - CYTHONIZE=1 MACOSX_DEPLOYMENT_TARGET=10.9 python setup.py build_ext -i + CYTHONIZE=1 python setup.py build_ext -i + + - name: Test + run: | + pytest --cov cyvcf2 --cov-report term-missing + + windows_build: + name: Run tests on Python windows-2022 MSYS2 UCRT64 + runs-on: windows-2022 + defaults: + run: + shell: msys2 {0} + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: "Setup MSYS2" + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + path-type: inherit + install: >- + mingw-w64-ucrt-x86_64-gcc + mingw-w64-ucrt-x86_64-make + mingw-w64-ucrt-x86_64-libdeflate + mingw-w64-ucrt-x86_64-xz + mingw-w64-ucrt-x86_64-curl + mingw-w64-ucrt-x86_64-zlib + mingw-w64-ucrt-x86_64-bzip2 + mingw-w64-ucrt-x86_64-tools-git + mingw-w64-ucrt-x86_64-python-pkgconfig + mingw-w64-ucrt-x86_64-pkg-config + mingw-w64-ucrt-x86_64-ninja + mingw-w64-ucrt-x86_64-python + mingw-w64-ucrt-x86_64-python-pip + make + automake + autoconf + git + + - name: Install Windows build prerequisites + run: | + cd htslib + autoreconf -i + ./configure --enable-libcurl --enable-s3 --enable-lzma --enable-bz2 --with-libdeflate + make + make install + + - name: Install + run: | + pip install -r requirements.txt + pip install pytest pytest-cov + CYTHONIZE=1 python setup.py build_ext -i - name: Test run: | pytest --cov cyvcf2 --cov-report term-missing sdist: - runs-on: ${{ matrix.os }}-latest + runs-on: ubuntu-22.04 strategy: matrix: - os: [ubuntu] python-version: ["3.12"] steps: diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 4c2848a..0f7513f 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -18,43 +18,71 @@ on: jobs: build_wheels: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }}-latest + name: Build wheels for ${{ matrix.python-version }}-${{ matrix.buildplat[1] }} + runs-on: ${{ matrix.buildplat[0] }} strategy: matrix: - os: [ubuntu, macos] - python-version: ["3.12"] + buildplat: + - [ubuntu-22.04, manylinux_x86_64] + - [ubuntu-22.04, musllinux_x86_64] + - [ubuntu-22.04, manylinux_aarch64] + - [ubuntu-22.04, musllinux_aarch64] + # macOS build arm64 on x86_64 with cross-compiler + - [macos-12, macosx_x86_64] + python-version: [pp310, cp37, cp38, cp39, cp310, cp311, cp312] + exclude: + # pp310, cp37, cp38 on musllinux is not support + - buildplat: [ubuntu-22.04, musllinux_x86_64] + python-version: cp37 + - buildplat: [ubuntu-22.04, musllinux_x86_64] + python-version: cp38 + - buildplat: [ubuntu-22.04, musllinux_x86_64] + python-version: pp310 + - buildplat: [ubuntu-22.04, musllinux_aarch64] + python-version: cp37 + - buildplat: [ubuntu-22.04, musllinux_aarch64] + python-version: cp38 + - buildplat: [ubuntu-22.04, musllinux_aarch64] + python-version: pp310 steps: - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - name: Set up QEMU if: runner.os == 'Linux' uses: docker/setup-qemu-action@v3 - - name: Build wheels for Linux + - name: Build wheels uses: pypa/cibuildwheel@v2.16.2 - if: runner.os == 'Linux' with: package-dir: . output-dir: wheelhouse config-file: "{package}/pyproject.toml" env: - CIBW_SKIP: "pp* *i686*" + # select + CIBW_BUILD: ${{ matrix.python-version }}-${{ matrix.buildplat[1] }} + + # linux CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 - CIBW_MUSLLINUX_X86_64_IMAGE: musllinux_1_1 + # manylinux2014 can't build on aarch64 + CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 + CIBW_MUSLLINUX_X86_64_IMAGE: musllinux_1_2 + CIBW_MUSLLINUX_AARCH64_IMAGE: musllinux_1_2 CIBW_ARCHS_LINUX: auto64 aarch64 CIBW_BEFORE_BUILD_LINUX: "{project}/ci/linux-deps" - CIBW_TEST_COMMAND: "{project}/ci/test" + CIBW_REPAIR_WHEEL_COMMAND_LINUX: 'LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib64" && auditwheel repair -w {dest_dir} {wheel}' + + # macos + CIBW_ARCHS_MACOS: auto64 arm64 + CIBW_BEFORE_BUILD_MACOS: "{project}/ci/osx-deps" + + # build CIBW_ENVIRONMENT: >- CYVCF2_HTSLIB_CONFIGURE_OPTIONS="--enable-libcurl --enable-s3 --enable-lzma --enable-bz2 --with-libdeflate" CYTHONIZE=1 + CIBW_TEST_COMMAND: "{project}/ci/test" # Enable tmate debugging of manually-triggered workflows if the input option was provided - name: Setup tmate session @@ -67,7 +95,7 @@ jobs: build_sdist: name: Build source distribution - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 with: @@ -90,7 +118,7 @@ jobs: upload_pypi: needs: [build_wheels, build_sdist] - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 # upload to PyPI on every tag starting with 'v' if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') steps: diff --git a/.gitignore b/.gitignore index ad219d6..c3def5c 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ _templates setup-requires/* .cache/v/cache/lastfailed .idea +**/__pycache__ diff --git a/ci/Dockerfile.alpine.test b/ci/Dockerfile.alpine.test new file mode 100644 index 0000000..e2b841e --- /dev/null +++ b/ci/Dockerfile.alpine.test @@ -0,0 +1,15 @@ +ARG PYTHON_VERSION=alpine + +FROM python:${PYTHON_VERSION} + +WORKDIR /workspace + +RUN apk add --no-cache build-base autoconf automake git xz-dev curl-dev libdeflate-dev bzip2-dev + +COPY . . + +RUN pip install -r requirements.txt && pip install pytest pytest-cov + +# build cyvcf2 +RUN CYVCF2_HTSLIB_CONFIGURE_OPTIONS="--enable-libcurl --enable-s3 --enable-lzma --enable-bz2 --with-libdeflate" \ + CYTHONIZE=1 python setup.py build_ext -i diff --git a/ci/Dockerfile.test b/ci/Dockerfile.slim.test similarity index 100% rename from ci/Dockerfile.test rename to ci/Dockerfile.slim.test diff --git a/ci/linux-deps b/ci/linux-deps index 7c2c71b..70e66fd 100755 --- a/ci/linux-deps +++ b/ci/linux-deps @@ -13,16 +13,15 @@ echo "manylinux image woking on $ID" # Install cyvcf2 development files. case "$ID" in -alpine) - apk add --no-cache xz-dev curl-dev +almalinux) + dnf install -y bzip2-devel xz-devel libcurl-devel openssl-devel openblas-devel epel-release - LIBDEFLATE_VERSION=1.19 - curl -L -o libdeflate-v"$LIBDEFLATE_VERSION".tar.gz https://github.com/ebiggers/libdeflate/archive/refs/tags/v"$LIBDEFLATE_VERSION".tar.gz - tar xzf libdeflate-v"$LIBDEFLATE_VERSION".tar.gz - cd libdeflate-"$LIBDEFLATE_VERSION" - cmake -B build && cmake --build build - cd ./build/ && make install - cd ../.. + # packages at epel + dnf install -y libdeflate-devel + ;; + +alpine) + apk add --no-cache xz-dev curl-dev libdeflate-dev ;; centos) diff --git a/ci/osx-arm64-deps b/ci/osx-arm64-deps deleted file mode 100755 index 4a4918f..0000000 --- a/ci/osx-arm64-deps +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -set -e - -git submodule init -git submodule update -git submodule update --init --recursive - -# configure fails with autoconf 2.71, so downgrade -# see https://github.com/asdf-vm/asdf-erlang/issues/195 -brew install autoconf@2.69 && \ -brew link --overwrite autoconf@2.69 && \ -autoconf -V - -cd htslib -autoheader -autoconf -./configure --enable-libcurl --enable-s3 --enable-lzma --enable-bz2 --without-libdeflate -make diff --git a/ci/osx-deps b/ci/osx-deps index b349908..907700e 100755 --- a/ci/osx-deps +++ b/ci/osx-deps @@ -1,26 +1,26 @@ #!/bin/bash -set -e +set -euo pipefail -git submodule init -git submodule update -git submodule update --init --recursive +export DYLD_LIBRARY_PATH=/usr/local/lib +# same with python +export MACOSX_DEPLOYMENT_TARGET=10.9 -# configure fails with autoconf 2.71, so downgrade -# see https://github.com/asdf-vm/asdf-erlang/issues/195 -brew install autoconf@2.69 && \ -brew link --overwrite autoconf@2.69 && \ -autoconf -V +brew install automake +brew unlink xz -curl -L -o libdeflate-v1.8.tar.gz https://github.com/ebiggers/libdeflate/archive/refs/tags/v1.8.tar.gz -tar xzf libdeflate-v1.8.tar.gz -cd libdeflate-1.8 -make -make install -cd .. +# build liblzma and libdelfate for muti arch -cd htslib -autoheader -autoconf -./configure --enable-libcurl --enable-s3 --enable-lzma --enable-bz2 -make +XZ_VERSION=5.4.5 +curl -L -o xz-$XZ_VERSION.tar.gz "https://github.com/tukaani-project/xz/releases/download/v$XZ_VERSION/xz-$XZ_VERSION.tar.gz" +tar -xf xz-$XZ_VERSION.tar.gz +cd xz-$XZ_VERSION +cmake -B build && cmake --build build +cd ./build && make install + +LIBDEFLATE_VERSION=1.19 +curl -L -o libdeflate-v"$LIBDEFLATE_VERSION".tar.gz https://github.com/ebiggers/libdeflate/archive/refs/tags/v"$LIBDEFLATE_VERSION".tar.gz +tar xzf libdeflate-v"$LIBDEFLATE_VERSION".tar.gz +cd libdeflate-"$LIBDEFLATE_VERSION" +cmake -B build && cmake --build build +cd ./build && make install diff --git a/cyvcf2/tests/test_reader.py b/cyvcf2/tests/test_reader.py index 13f3f9e..0b466e5 100644 --- a/cyvcf2/tests/test_reader.py +++ b/cyvcf2/tests/test_reader.py @@ -1,5 +1,6 @@ from __future__ import print_function import os.path +import platform import tempfile import sys import os @@ -965,13 +966,20 @@ def test_alt_homozygous_gt(): assert v.gt_bases[0] == '<*:DEL>/<*:DEL>' def test_write_missing_contig(): - input_vcf = VCF('{}/seg.vcf.gz'.format(HERE)) - output_vcf = Writer('/dev/null', input_vcf) + input_vcf = VCF("{}/seg.vcf.gz".format(HERE)) + if platform.system() == "Windows": + output_vcf = "__o.vcf" + else: + output_vcf = "/dev/null" + output_vcf = Writer(output_vcf, input_vcf) for v in input_vcf: - v.genotypes = [[1,1,False]] + v.genotypes = [[1, 1, False]] output_vcf.write_record(v) output_vcf.close() + if platform.system() == "Windows": + os.unlink("__o.vcf") + def test_set_samples(): vcf = VCF(VCF_PATH) assert len(vcf.samples) == 189, len(vcf.samples) @@ -1000,9 +1008,14 @@ def test_issue44(): # "./." "." ".|." "0|0" expected = [[-1, -1, False], [-1, False], [-1, -1, True], [0, 0, True]] #print("", file=sys.stderr) - for i, v in enumerate(VCF('__o.vcf')): + + t = VCF('__o.vcf') + for i, v in enumerate(t): #print(v.genotypes, file=sys.stderr) assert v.genotypes == [expected[i]], (i, v.genotypes, expected[i]) + + # file should be closed before delete + t.close() os.unlink("__o.vcf") def test_id_field_updates(): diff --git a/htslib b/htslib index d8ca374..8f72310 160000 --- a/htslib +++ b/htslib @@ -1 +1 @@ -Subproject commit d8ca374b1977601246ce4a20370c7b78441a01aa +Subproject commit 8f7231035d0409d525767c66d9f49f1f967ee1df diff --git a/pyproject.toml b/pyproject.toml index 8193b5d..4018caa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,8 @@ [build-system] -requires = ["setuptools", "wheel", "cython>=0.23.3", "oldest-supported-numpy"] +requires = [ + "setuptools", + "wheel", + "cython>=0.23.3", + 'oldest-supported-numpy; os_name != "nt"', + 'numpy; os_name == "nt"' +] diff --git a/requirements.txt b/requirements.txt index 91828fc..4b1062f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ cython>=0.23.3 -oldest-supported-numpy coloredlogs click setuptools +numpy; os_name == 'nt' +oldest-supported-numpy; os_name != 'nt' diff --git a/setup.py b/setup.py index 5e927c0..b855fe4 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ import ctypes import glob import os +import platform import shutil import sys import subprocess @@ -50,16 +51,18 @@ def no_cythonize(extensions, **_ignore): def check_libhts(): - try: - ctypes.CDLL("libhts.so") - return True - except Exception: - return False - + os_type = platform.system() + if os_type == "Linux": + lib_name = "libhts.so" + elif os_type == "Darwin": # macOS + lib_name = "libhts.dylib" + elif os_type == "Windows": + lib_name = "hts-3.dll" + else: + return False # Unsupported OS -def check_libdeflate(): try: - ctypes.CDLL("libdeflate.so") + ctypes.CDLL(lib_name) return True except Exception: return False @@ -106,6 +109,12 @@ class cyvcf2_build_ext(build_ext): def run(self): print("# cyvcf2: htslib mode is {}".format(CYVCF2_HTSLIB_MODE)) if CYVCF2_HTSLIB_MODE == "BUILTIN" or not check_libhts(): + if platform.system() == "Windows": + # Windows htslib can't be built internall + raise RuntimeError( + "Required library 'hts-3.dll' not found. Please install htslib first." + ) + print( "# cyvcf2: htslib configure options is {}".format( CYVCF2_HTSLIB_CONFIGURE_OPTIONS @@ -129,7 +138,7 @@ def run(self): extra_libs = [] # read the htslib config.status file to get the linked libraries - with open("htslib/config.status", "r") as f: + with open(os.path.join("htslib", "config.status"), "r") as f: for line in f: if 'S["static_LIBS"]' in line: linked_libs_str = line.split("=")[1].strip()[1:-1] @@ -146,6 +155,9 @@ def run(self): class cyvcf2_sdist(sdist): def run(self): + if platform.system() == "Windows": + raise RuntimeError("cyvcf2 Can't build sdist on Windows") + pre_sdist() super().run() @@ -174,15 +186,16 @@ def run(self): if os.path.exists(cyvcf2_c_path): os.remove(cyvcf2_c_path) - so_files = glob.glob("cyvcf2/cyvcf2.cpython-*.so") - for file in so_files: + lib_files = glob.glob("cyvcf2/cyvcf2.cpython-*") + for file in lib_files: os.remove(file) - print("cleaning htslib build files") - current_directory = os.getcwd() - os.chdir(os.path.join(current_directory, "htslib")) - subprocess.run(["make", "distclean"], check=True) - os.chdir(current_directory) + if platform.system() != "Windows": + print("cleaning htslib build files") + current_directory = os.getcwd() + os.chdir(os.path.join(current_directory, "htslib")) + subprocess.run(["make", "distclean"], check=True) + os.chdir(current_directory) # How to link against HTSLIB @@ -190,7 +203,20 @@ def run(self): # builtin htslib code (default) # EXTERNAL: use shared libhts.so compiled outside of # cyvcf2 -CYVCF2_HTSLIB_MODE = os.environ.get("CYVCF2_HTSLIB_MODE", "BUILTIN") +if platform.system() == "Windows": + # can't static link to htslib on Windows + htslib_mode_default = "EXTERNAL" +else: + htslib_mode_default = "BUILTIN" + +CYVCF2_HTSLIB_MODE = os.environ.get("CYVCF2_HTSLIB_MODE", htslib_mode_default) + +if platform.system() == "Windows" and CYVCF2_HTSLIB_MODE == "BUILTIN": + print( + "# cyvcf2 WARNING: The use of cyvcf2 on Windows is experimental. It will not work when statically linked to htslib. Fallback to htslib EXTERNAL mode" + ) + CYVCF2_HTSLIB_MODE = "EXTERNAL" + CYVCF2_HTSLIB_CONFIGURE_OPTIONS = os.environ.get( "CYVCF2_HTSLIB_CONFIGURE_OPTIONS", None )