fix some errors in github action, and add python 3.12 (#286)

* fix build broken on cython 3.0.x * remove error StopIteration * add python 3.12 * add setuptools for python 3.12 * add github actions, build and test in python offical docker image on arm64 and amd64 * fix core dump after seg vcf write
brentp · Dec 10, 2023 · eca1d90 · eca1d90
1 parent c1a2dba
commit eca1d90
Show file tree

Hide file tree

Showing 6 changed files with 93 additions and 40 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -3,12 +3,45 @@ name: Build
 on: [push, pull_request]
 
 jobs:
+  docker-build-and-test:
+    name: Docker Build and Test on ${{ matrix.platform }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm64
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Docker Build and Push
+        uses: docker/build-push-action@v5
+        with:
+          context: .  
+          file: ./ci/Dockerfile.test
+          platforms: ${{ matrix.platform }}
+          tags: cyvcf2:test
+          push: false
+          load: true
+          build-args: |  
+            PYTHON_VERSION=slim
+
+      - name: Docker Run Tests
+        run: |
+          docker run --rm --platform ${{ matrix.platform }} cyvcf2:test pytest --cov cyvcf2 --cov-report term-missing
+
   build:
     name: Run tests on Python ${{ matrix.python-version }}
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v3

diff --git a/ci/Dockerfile.test b/ci/Dockerfile.test
@@ -0,0 +1,20 @@
+ARG PYTHON_VERSION=slim 
+
+FROM python:${PYTHON_VERSION}
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install --no-install-recommends -y autoconf automake gcc \
+    libcurl4-openssl-dev make git libc6-dev zlib1g-dev libssl-dev liblzma-dev libbz2-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY . .
+
+# build htslib
+RUN cd htslib && autoheader && autoconf && autoreconf --install && \
+    ./configure --enable-s3 --disable-lzma --disable-bz2 && make
+
+RUN pip install -r requirements.txt && pip install pytest pytest-cov
+
+# build cyvcf2
+RUN CYTHONIZE=1 python setup.py build_ext -i
diff --git a/cyvcf2/cyvcf2.pxd b/cyvcf2/cyvcf2.pxd
@@ -93,36 +93,36 @@ cdef extern from "htslib/vcf.h":
     hts_itr_t *bcf_itr_querys(hts_idx_t *, void *, char *);
 
 
-    const int BCF_DT_ID = 0;
-    const int BCF_DT_CTG = 1
-    const int BCF_DT_SAMPLE = 2;
+    cdef extern const int BCF_DT_ID;
+    cdef extern const int BCF_DT_CTG;
+    cdef extern const int BCF_DT_SAMPLE;
 
-    uint32_t bcf_float_missing = 0x7F800001;
+    cdef extern uint32_t bcf_float_missing;
 
-    const int BCF_ERR_CTG_UNDEF = 1
+    cdef extern const int BCF_ERR_CTG_UNDEF;
 
 
-    const int BCF_BT_NULL   = 0
-    const int BCF_BT_INT8   = 1
-    const int BCF_BT_INT16  = 2
-    const int BCF_BT_INT32  = 3
-    const int BCF_BT_FLOAT  = 5
-    const int BCF_BT_CHAR   = 7
+    cdef extern const int BCF_BT_NULL;
+    cdef extern const int BCF_BT_INT8;
+    cdef extern const int BCF_BT_INT16;
+    cdef extern const int BCF_BT_INT32;
+    cdef extern const int BCF_BT_FLOAT;
+    cdef extern const int BCF_BT_CHAR;
 
-    const int bcf_str_missing = 0x07
-    const int bcf_str_vector_end = 0
+    cdef extern const int bcf_str_missing;
+    cdef extern const int bcf_str_vector_end;
 
-    const int INT8_MIN = -128
-    const int INT16_MIN = -32768
-    const int INT32_MIN = -2147483648
+    cdef extern const int INT8_MIN;
+    cdef extern const int INT16_MIN;
+    cdef extern const int INT32_MIN;
 
-    const int bcf_int8_vector_end  = -127
-    const int bcf_int16_vector_end  = -32767
-    const int bcf_int32_vector_end  = -2147483647
+    cdef extern const int bcf_int8_vector_end;
+    cdef extern const int bcf_int16_vector_end;
+    cdef extern const int bcf_int32_vector_end;
 
-    const int bcf_int8_missing  = INT8_MIN 
-    const int bcf_int16_missing  = INT16_MIN 
-    const int32_t bcf_int32_missing  = INT32_MIN
+    cdef extern const int bcf_int8_missing;
+    cdef extern const int bcf_int16_missing;
+    cdef extern const int32_t bcf_int32_missing;
 
     ctypedef union uv1:
         int32_t i; # integer value
@@ -179,17 +179,17 @@ cdef extern from "htslib/vcf.h":
     ctypedef struct bcf_idpair_t:
         pass
 
-    const int BCF_HL_FLT  = 0 # header line
-    const int BCF_HL_INFO = 1
-    const int BCF_HL_FMT  = 2
-    const int BCF_HL_CTG  = 3
-    const int BCF_HL_STR  = 4 # structured header line TAG=<A=..,B=..>
-    const int BCF_HL_GEN  = 5 # generic header line
-
-    const int BCF_HT_FLAG = 0 # header type
-    const int BCF_HT_INT  = 1
-    const int BCF_HT_REAL = 2
-    const int BCF_HT_STR  = 3
+    cdef extern const int BCF_HL_FLT; # header line
+    cdef extern const int BCF_HL_INFO;
+    cdef extern const int BCF_HL_FMT;
+    cdef extern const int BCF_HL_CTG;
+    cdef extern const int BCF_HL_STR; # structured header line TAG=<A=..,B=..>
+    cdef extern const int BCF_HL_GEN; # generic header line
+
+    cdef extern const int BCF_HT_FLAG; # header type
+    cdef extern const int BCF_HT_INT;
+    cdef extern const int BCF_HT_REAL;
+    cdef extern const int BCF_HT_STR;
 
     ctypedef struct bcf_hrec_t:
         int type;       # One of the BCF_HL_* type

diff --git a/cyvcf2/cyvcf2.pyx b/cyvcf2/cyvcf2.pyx
@@ -407,7 +407,7 @@ cdef class VCF(HTSFile):
         itr = bcf_itr_querys(self.hidx, self.hdr, to_bytes(region))
         if itr == NULL:
             warnings.warn("no intervals found for %s at %s" % (self.fname, region))
-            raise StopIteration
+            return
         try:
             while True:
                 b = bcf_init()
@@ -437,11 +437,11 @@ cdef class VCF(HTSFile):
         """
         if not region:
             yield from self
-            raise StopIteration
+            return
 
         if self.fname.decode(ENC).endswith(('.bcf', '.bcf.gz')):
             yield from self._bcf_region(region)
-            raise StopIteration
+            return
 
         if self.idx == NULL:
             self.idx = tbx_index_load(to_bytes(self.fname))
@@ -459,7 +459,7 @@ cdef class VCF(HTSFile):
 
         if itr == NULL:
             warnings.warn("no intervals found for %s at %s" % (self.fname, region))
-            raise StopIteration
+            return
 
         try:
             while 1:
@@ -2404,7 +2404,7 @@ cdef class Writer(VCF):
         if not self.header_written:
             self.write_header()
         if var.b.errcode == BCF_ERR_CTG_UNDEF:
-            h = bcf_hdr_id2hrec(self.ohdr, BCF_DT_CTG, 0, var.b.rid)
+            h = bcf_hdr_id2hrec(bcf_hdr_dup(self.ohdr), BCF_DT_CTG, 0, var.b.rid)
             if h == NULL:
                 raise Exception("contig %d unknown and not found in header" % var.b.rid)
             if bcf_hdr_add_hrec(self.hdr, h) < 0:

diff --git a/requirements.txt b/requirements.txt
@@ -2,3 +2,4 @@ cython>=0.23.3
 numpy
 coloredlogs
 click
+setuptools
diff --git a/setup.py b/setup.py
@@ -4,7 +4,6 @@
 import subprocess
 import platform
 
-import pkg_resources
 from setuptools import setup, Extension
 
 if sys.version_info.major == 2 and sys.version_info.minor != 7: