Support CPython 3.11, 3.12, and aarch64 processors

.github/workflows/pythonpackage.yml

@@ -17,8 +17,8 @@ defaults:
     shell: bash -l {0}
 
 concurrency:
-  group: ${{ github.ref }}
-  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   main:
@@ -30,8 +30,8 @@ jobs:
       fail-fast: false
       max-parallel: 12
       matrix:
-        os: [ubuntu-latest, windows-latest, macOS-11]
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        os: [ubuntu-latest, windows-latest, macOS-12]
+        python-version: [3.8, 3.9]
 
     steps:
     - name: disk space
@@ -40,9 +40,9 @@ jobs:
         df -h
     - uses: maxim-lobanov/setup-xcode@v1
       # alternative would be to upgrade tapi to 1100.0.11, we can possibly remove this in the future
-      if: matrix.os == 'macOS-11'
+      if: matrix.os == 'macOS-12'
       with:
-        xcode-version: "11.7.0"
+        xcode-version: "13.2.1"
     - uses: actions/checkout@v2
       with:
         submodules: true
@@ -53,11 +53,12 @@ jobs:
         token: ${{ secrets.PAT_PULL_ENTERPRISE }}
         path: vaex-enterprise
     - name: install micromamba
-      uses: mamba-org/provision-with-micromamba@main
+      uses: mamba-org/setup-micromamba@v1
       with:
+        micromamba-version: 1.1.0-1
         environment-name: vaex-dev
         environment-file: ci/conda-env.yml
-        extra-specs: |
+        create-args: >-
             python=${{ matrix.python-version }}
     - name: Extra non-windows installs
       if: matrix.os != 'windows-latest'
@@ -79,12 +80,12 @@ jobs:
     #     docker rmi $(docker image ls -aq)
     #     df -h
     - name: Install OpenMP runtime (Mac-only)
-      if: matrix.os == 'macOS-11'
+      if: matrix.os == 'macOS-12'
       run: |
         brew install libomp
     - name: Cache compiled binaries
       # this fails for this combination, leading to binaries filled with 0's
-      if: matrix.python-version != '3.6' || matrix.os != 'macOS-11'
+      if: matrix.os != 'macOS-12'
       id: cache-compiled-binaries
       uses: actions/cache@v2
       with:
@@ -137,11 +138,6 @@ jobs:
     #     # flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
     #     # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
     #     # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Install fixes for Python 3.6
-      if: matrix.python-version == '3.6'
-      run: |
-        # by default on 3.6 we get an old version, so manually upgrade
-        pip install gcsfs==0.8.0
     - name: Install pytest-asyncio correct version
       if: matrix.python-version != '3.10'
       run: |
@@ -155,8 +151,8 @@ jobs:
       run: |
         VAEX_CACHE_RESULTS=1 ./ci/04-run-test-suite.sh
     - name: Check ml spec
-      # no catboost for py39 and py37
-      if: matrix.python-version != '3.9' && matrix.os != 'windows-latest' && matrix.python-version != '3.7'
+      # no catboost for py39 and py37, xgboost fails on 3.6
+      if: matrix.python-version != '3.9' && matrix.os != 'windows-latest'
       run: |
         python -m vaex.ml.spec packages/vaex-ml/vaex/ml/spec_new.json
         diff packages/vaex-ml/vaex/ml/spec_new.json packages/vaex-ml/vaex/ml/spec.json
@@ -165,19 +161,19 @@ jobs:
 #      run: |
 #        py.test vaex-enterprise/tests --timeout=1000
     - name: Test notebooks
-      if: matrix.os != 'windows-latest' && matrix.python-version != '3.6'
+      if: matrix.os != 'windows-latest'
       run: |
         ./ci/05-run-notebooks.sh
     - name: Authenticate Google Cloud Platform
-      if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') && !((matrix.os == 'windows-latest') || (matrix.os == 'macOS-11' && matrix.python-version == '3.6'))  }}
+      if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') && !((matrix.os == 'windows-latest'))  }}
       uses: google-github-actions/setup-gcloud@v0
       with:
         project_id: ${{ secrets.GCP_PROJECT_ID_VAEX }}
         service_account_key: ${{ secrets.GCP_SA_KEY_VAEX }}
         export_default_credentials: true
     - name: Test vaex-contrib
-      # do not run in a PR from someone else, skip windows, and osx+py36
-      if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') && !((matrix.os == 'windows-latest') || (matrix.os == 'macOS-11' && matrix.python-version == '3.6'))  }}
+      # do not run in a PR from someone else, skip windows
+      if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') && !((matrix.os == 'windows-latest'))  }}
       env:
         PROJECT_ID: ${{ secrets.GCP_PROJECT_ID_VAEX }}
       run: |
@@ -191,12 +187,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macOS-11]
-        python-version: [3.7, 3.8, 3.9]
-        # ssl/certifi issues with this combination
-        exclude:
-          - os: windows-latest
-            python-version: 3.7
+        os: [ubuntu-latest, windows-latest, macOS-12]
+        python-version: [3.8, 3.9]
 
     steps:
     - uses: actions/checkout@v2
@@ -206,11 +198,12 @@ jobs:
         path: ./dist
 
     - name: Install micromamba
-      uses: mamba-org/provision-with-micromamba@main
+      uses: mamba-org/setup-micromamba@v1
       with:
+        micromamba-version: 1.5.6-0
         environment-name: vaex-test
         environment-file: ci/conda-base-minimal.yml
-        extra-specs: |
+        create-args: >-
             python=${{ matrix.python-version }}
             pcre
             rich

.github/workflows/wheel-universal.yml

@@ -2,16 +2,17 @@ name: Build universal wheel
 
 on:
   push:
-    tags:
-      - meta-*
-      - astro-*
-      - graphql-*
-      - jupyter-*
-      - ml-*
-      - viz-*
-      - hdf5-*
-      - server-*
-      - contrib-*
+    branches:
+      - master
+  pull_request:
+    branches:
+    - master
+  schedule:
+  - cron: "0 2 * * 1-5"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   build_wheels:
@@ -41,7 +42,6 @@ jobs:
         mkdir dist
 
     - name: Build vaex (meta)
-      if: startsWith(github.ref, 'refs/tags/meta')
       run: |
         (cp README.md packages/vaex-meta/ && cd packages/vaex-meta && python setup.py sdist bdist_wheel); cp packages/vaex-meta/dist/* dist
 
@@ -54,7 +54,6 @@ jobs:
         twine upload dist/vaex* dist/vaex*
 
     - name: Build vaex-graphql
-      if: startsWith(github.ref, 'refs/tags/graphql')
       run: |
         (cd packages/vaex-graphql && python setup.py sdist bdist_wheel); cp packages/vaex-graphql/dist/* dist
 
@@ -68,7 +67,6 @@ jobs:
 
 
     - name: Build vaex-jupyter
-      if: startsWith(github.ref, 'refs/tags/jupyter')
       run: |
         (cd packages/vaex-jupyter && python setup.py sdist bdist_wheel); cp packages/vaex-jupyter/dist/* dist
 
@@ -81,7 +79,6 @@ jobs:
         twine upload dist/vaex-jupyter* dist/vaex_jupyter*
 
     - name: Build vaex-ml
-      if: startsWith(github.ref, 'refs/tags/ml')
       run: |
         (cd packages/vaex-ml && python setup.py sdist bdist_wheel); cp packages/vaex-ml/dist/* dist
 
@@ -94,7 +91,6 @@ jobs:
         twine upload dist/vaex-ml* dist/vaex_ml*
 
     - name: Build vaex-contrib
-      if: startsWith(github.ref, 'refs/tags/contrib')
       run: |
         (cd packages/vaex-contrib && python setup.py sdist bdist_wheel); cp packages/vaex-contrib/dist/* dist
 
@@ -109,7 +105,6 @@ jobs:
 
 
     - name: Build vaex-viz
-      if: startsWith(github.ref, 'refs/tags/viz')
       run: |
         (cd packages/vaex-viz && python setup.py sdist bdist_wheel); cp packages/vaex-viz/dist/* dist
 
@@ -122,7 +117,6 @@ jobs:
         twine upload dist/vaex-viz* dist/vaex_viz*
 
     - name: Build vaex-astro
-      if: startsWith(github.ref, 'refs/tags/astro')
       run: |
         (cd packages/vaex-astro && python setup.py sdist bdist_wheel); cp packages/vaex-astro/dist/* dist
 
@@ -135,7 +129,6 @@ jobs:
         twine upload dist/vaex-astro* dist/vaex_astro*
 
     - name: Build vaex-hdf5
-      if: startsWith(github.ref, 'refs/tags/hdf5')
       run: |
         (cd packages/vaex-hdf5 && python setup.py sdist bdist_wheel); cp packages/vaex-hdf5/dist/* dist
 
@@ -148,7 +141,6 @@ jobs:
         twine upload dist/vaex-hdf5* dist/vaex_hdf5*
 
     - name: Build vaex-server
-      if: startsWith(github.ref, 'refs/tags/server')
       run: |
         (cd packages/vaex-server && python setup.py sdist bdist_wheel); cp packages/vaex-server/dist/* dist