Support CPython 3.11, 3.12, and aarch64 processors

.github/workflows/pythonpackage.yml

@@ -42,7 +42,7 @@ jobs:
       # alternative would be to upgrade tapi to 1100.0.11, we can possibly remove this in the future
       if: matrix.os == 'macOS-11'
       with:
-        xcode-version: "11.7.0"
+        xcode-version: "13.2.1"
     - uses: actions/checkout@v2
       with:
         submodules: true
@@ -53,11 +53,12 @@ jobs:
         token: ${{ secrets.PAT_PULL_ENTERPRISE }}
         path: vaex-enterprise
     - name: install micromamba
-      uses: mamba-org/provision-with-micromamba@main
+      uses: mamba-org/setup-micromamba@v1
       with:
+        micromamba-version: 1.5.6-0
         environment-name: vaex-dev
         environment-file: ci/conda-env.yml
-        extra-specs: |
+        create-args: >-
             python=${{ matrix.python-version }}
     - name: Extra non-windows installs
       if: matrix.os != 'windows-latest'
@@ -155,8 +156,8 @@ jobs:
       run: |
         VAEX_CACHE_RESULTS=1 ./ci/04-run-test-suite.sh
     - name: Check ml spec
-      # no catboost for py39 and py37
-      if: matrix.python-version != '3.9' && matrix.os != 'windows-latest' && matrix.python-version != '3.7'
+      # no catboost for py39 and py37, xgboost fails on 3.6
+      if: matrix.python-version != '3.6' && matrix.python-version != '3.7' && matrix.python-version != '3.9' && matrix.os != 'windows-latest'
       run: |
         python -m vaex.ml.spec packages/vaex-ml/vaex/ml/spec_new.json
         diff packages/vaex-ml/vaex/ml/spec_new.json packages/vaex-ml/vaex/ml/spec.json
@@ -165,7 +166,7 @@ jobs:
 #      run: |
 #        py.test vaex-enterprise/tests --timeout=1000
     - name: Test notebooks
-      if: matrix.os != 'windows-latest' && matrix.python-version != '3.6'
+      if: matrix.os != 'windows-latest' && matrix.python-version != '3.6' && matrix.python-version != '3.7'
       run: |
         ./ci/05-run-notebooks.sh
     - name: Authenticate Google Cloud Platform
@@ -206,11 +207,12 @@ jobs:
         path: ./dist
 
     - name: Install micromamba
-      uses: mamba-org/provision-with-micromamba@main
+      uses: mamba-org/setup-micromamba@v1
       with:
+        micromamba-version: 1.5.6-0
         environment-name: vaex-test
         environment-file: ci/conda-base-minimal.yml
-        extra-specs: |
+        create-args: >-
             python=${{ matrix.python-version }}
             pcre
             rich

.github/workflows/wheel.yml

@@ -4,86 +4,111 @@ on:
   push:
     tags:
       - core-*
+  release:
+    types: [released, prereleased]
+  workflow_dispatch:  # allows running workflow manually from the Actions tab
 
 jobs:
+  build_wheels_matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      include: ${{ steps.set-matrix.outputs.include }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      - run: pip install cibuildwheel==2.17.0 # sync version with pypa/cibuildwheel below
+
+      - id: set-matrix
+        run: |
+          MATRIX_INCLUDE=$(
+            {
+              cibuildwheel --print-build-identifiers --platform linux --arch x86_64,aarch64 | grep cp |  jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \
+              && cibuildwheel --print-build-identifiers --platform macos --arch x86_64 | grep cp |  jq -nRc '{"only": inputs, "os": "macos-13"}' \
+              && cibuildwheel --print-build-identifiers --platform macos --arch arm64 | grep cp |  jq -nRc '{"only": inputs, "os": "macos-14"}' \
+              && cibuildwheel --print-build-identifiers --platform windows --arch AMD64 | grep cp |  jq -nRc '{"only": inputs, "os": "windows-latest"}'
+            } | jq -sc
+          )
+          echo "include=$MATRIX_INCLUDE" >> $GITHUB_OUTPUT
+          cat $GITHUB_OUTPUT
+        working-directory: packages/vaex-core/
+
   build_wheels:
-    name: Build wheel on ${{ matrix.os }}
+    needs: build_wheels_matrix
     runs-on: ${{ matrix.os }}
+    name: Build ${{ matrix.only }}
+
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, macOS-11, windows-latest]
-        python-version: [36, 37, 38, 39, 310]
+        include: ${{ fromJson(needs.build_wheels_matrix.outputs.include) }}
+
+    permissions:
+      contents: write # for uploading release assets
 
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
       with:
         submodules: true
 
     - name: Copy dll
-      if: (matrix.os == 'windows-latest')
+      if: matrix.os == 'windows-latest'
       uses: ./ci/actions/windll
 
-    - uses: actions/setup-python@v2
-      name: Install Python
-      with:
-        python-version: '3.7'
-
-    - name: Install cibuildwheel
-      run: |
-        python -m pip install cibuildwheel==2.8.1
-
-    - name: Install twine/wheel
-      run: |
-        python -m pip install twine wheel
-
     - name: chores
       if: matrix.os != 'windows-latest'
       run: |
-        mkdir dist
         mkdir packages/vaex-core/bin
         cp bin/install_pcre.sh packages/vaex-core/bin/
 
-    - name: Build vaex-core (windows)
-      if: (matrix.os == 'windows-latest')
-      env:
-        CIBW_BUILD: cp${{ matrix.python-version }}*64
-        CIBW_ENVIRONMENT_LINUX: "CFLAGS='-Wl,-strip-all' CXXFLAGS='-Wl,-strip-all'"
-      run: |
-        python -m cibuildwheel --output-dir dist packages/vaex-core
+    - name: Set up QEMU
+      if: runner.os == 'Linux'
+      uses: docker/setup-qemu-action@v3
 
-    - name: Build vaex-core (linux, osx)
-      if: (matrix.os != 'windows-latest')
+    - uses: pypa/[email protected] # sync version with pip install cibuildwheel above
+      with:
+        only: ${{ matrix.only }}
+        package-dir: packages/vaex-core/
+        output-dir: packages/vaex-core/dist/
       env:
-        CIBW_BEFORE_BUILD: bin/install_pcre.sh
-        CIBW_BUILD: "cp${{ matrix.python-version }}*64 cp${{ matrix.python-version }}-macosx-*"
-        CIBW_ENVIRONMENT_LINUX: "CFLAGS='-Wl,-strip-all' CXXFLAGS='-Wl,-strip-all'"
+        CIBW_BEFORE_BUILD: ${{ ( matrix.os != 'windows-latest' && 'bin/install_pcre.sh' ) || '' }}
         CIBW_BUILD_VERBOSITY: 2
-        # needed for m1 builds
+        CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28
+        CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
+        CIBW_ENVIRONMENT_LINUX: "CFLAGS='-Wl,-strip-all' CXXFLAGS='-Wl,-strip-all'"
         CIBW_ENVIRONMENT_MACOS: "CFLAGS='-I/usr/local/include -L/usr/local/lib' CXXFLAGS='-I/usr/local/include -L/usr/local/lib' LDFLAGS='-L/usr/local/lib'"
-        CIBW_ARCHS_MACOS: "x86_64 arm64"
-      run: |
-        python -m cibuildwheel --output-dir dist packages/vaex-core
 
-    - name: Make source distribution
-      shell: bash
-      run: |
-          cd packages/vaex-core && python setup.py sdist
+    - name: Upload release assets
+      if: github.event_name == 'release'
+      uses: softprops/[email protected]
+      with:
+        files: packages/vaex-core/dist/*
 
-    - name: Copy build artifacts
-      shell: bash
-      run: |
-          cp packages/vaex-core/dist/* dist/
+    - uses: actions/setup-python@v5
+      with:
+        python-version: "3.x"
+
+    - name: Make source distribution
+      run: python setup.py sdist
+      working-directory: packages/vaex-core/
 
-    - uses: actions/upload-artifact@v1
+    - uses: actions/upload-artifact@v4
       with:
-        name: distributions
-        path: ./dist
+        name: distribution-${{ matrix.only }}
+        path: packages/vaex-core/dist
 
     - name: Publish a Python distribution to PyPI
       env:
         TWINE_USERNAME: __token__
         TWINE_PASSWORD: ${{ secrets.pypi_password_vaex_core }}
       run: |
+        pip install twine wheel
         openssl sha256 dist/*
-        twine upload --skip-existing dist/vaex?core*
+        twine upload --skip-existing
+      working-directory: packages/vaex-core/

bin/webveax

@@ -12,10 +12,10 @@ import sys
 import time
 import itertools
 import platform
+from importlib.machinery import SourceFileLoader
 
 
 import vaex.dataset
-import imp
 
 
 #import matplotlib
@@ -57,7 +57,7 @@ custom_module = None
 for path in path_list:
 	if os.path.exists(path):
 		print "loading", path
-		custom_module = imp.load_source('vaex.web.custom', path)
+		custom_module = SourceFileLoader('vaex.web.custom', path).load_module()
 	else:
 		print >>sys.stderr, path, "does not exist"
 if custom_module:
@@ -244,4 +244,4 @@ def api_density_xy(name, x, y=None):
 if __name__ == '__main__':
 	#flask.url_for('static', filename='test.html')
 	app.run(host='0.0.0.0')
-	#api_density_xy("Aq-A-2-999-shuffled-1percent", "x", "y")
+	#api_density_xy("Aq-A-2-999-shuffled-1percent", "x", "y")

ci/05-run-notebooks.sh

@@ -22,8 +22,8 @@ cd docs/source
  python -m nbconvert --TagRemovePreprocessor.remove_cell_tags="('skip-ci',)" --to html --execute guides/io.ipynb --ExecutePreprocessor.timeout=240
  python -m nbconvert --TagRemovePreprocessor.remove_cell_tags="('skip-ci',)" --to html --execute guides/ml_iris.ipynb --ExecutePreprocessor.timeout=240
  python -m nbconvert --TagRemovePreprocessor.remove_cell_tags="('skip-ci',)" --to html --execute guides/ml_titanic.ipynb --ExecutePreprocessor.timeout=240
- python -m nbconvert --TagRemovePreprocessor.remove_cell_tags="('skip-ci',)" --to html --execute example_jupyter_plotly.ipynb --ExecutePreprocessor.timeout=240
- python -m nbconvert --TagRemovePreprocessor.remove_cell_tags="('skip-ci',)" --to html --execute example_jupyter_ipyvolume.ipynb --ExecutePreprocessor.timeout=240
+#  python -m nbconvert --TagRemovePreprocessor.remove_cell_tags="('skip-ci',)" --to html --execute example_jupyter_plotly.ipynb --ExecutePreprocessor.timeout=240
+#  python -m nbconvert --TagRemovePreprocessor.remove_cell_tags="('skip-ci',)" --to html --execute example_jupyter_ipyvolume.ipynb --ExecutePreprocessor.timeout=240
 
 # this does not seem to work on osx:
 # # make sure the ~/.ipython dir exists because multiple processes might try to create it

ci/conda-env.yml

@@ -10,6 +10,7 @@ dependencies:
 - cython
 - cachetools
 - catboost
+- dask<2024.2.0
 - diskcache
 - filelock
 - fsspec<2022.2.0
@@ -20,7 +21,6 @@ dependencies:
 - h5py
 - httpx # for testing with starlette/fastapi
 - ipyvolume=0.6.0a6
-- lightgbm
 - matplotlib-base
 - nest-asyncio<1.5.2
 - notebook
@@ -54,3 +54,5 @@ dependencies:
 - python-utils
 - progressbar2
 - zipp<3.16.0
+- pip:
+  - lightgbm>=4.0.0

docs/source/conf.py

@@ -89,7 +89,7 @@
 	print("failed finding vaex module, try finding version")
 	import sys
 	import os
-	import imp
+	from importlib.machinery import SourceFileLoader
 
 	def system(cmd):
 		print("Executing: ", cmd)
@@ -102,7 +102,7 @@ def system(cmd):
 	if not os.path.exists(path_version_file):
 		system("version=`git describe --tags --long`; python/vaex/vaex/setversion.py ${version}")
 
-	version = imp.load_source('version', path_version_file)
+	version = SourceFileLoader('version', path_version_file).load_module()
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/source/datasets.ipynb

@@ -31,13 +31,13 @@
     "One can also stream the data directly from S3. Only the data that is necessary will be streamed, and it will cached locally:\n",
     "```\n",
     "import vaex\n",
-    "df = vaex.open('s3://vaex/taxi/yellow_taxi_2015_f32s.hdf5?anon=true')\n",
+    "df = vaex.open('s3://vaex/taxi/nyc_taxi_2015_mini.hdf5?anon=true')\n",
     "```"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2020-06-04T13:46:26.924698Z",

docs/source/tutorial.ipynb

@@ -799,7 +799,7 @@
    ],
    "source": [
     "# Read in the NYC Taxi dataset straight from S3\n",
-    "nyctaxi = vaex.open('s3://vaex/taxi/yellow_taxi_2009_2015_f32.hdf5?anon=true')\n",
+    "nyctaxi = vaex.open('s3://vaex/taxi/nyc_taxi_2015_mini.hdf5?anon=true')\n",
     "nyctaxi.head(5)"
    ]
   },

packages/vaex-arrow/setup.py

@@ -1,10 +1,10 @@
 import os
-import imp
 from setuptools import setup
+from importlib.machinery import SourceFileLoader
 
 dirname = os.path.dirname(__file__)
 path_version = os.path.join(dirname, "vaex_arrow/_version.py")
-version = imp.load_source('version', path_version)
+version = SourceFileLoader('version', path_version).load_module()
 
 
 name = 'vaex'

packages/vaex-astro/setup.py

@@ -1,10 +1,10 @@
 import os
-import imp
 from setuptools import setup
+from importlib.machinery import SourceFileLoader
 
 dirname = os.path.dirname(__file__)
 path_version = os.path.join(dirname, 'vaex/astro/_version.py')
-version = imp.load_source('version', path_version)
+version = SourceFileLoader('version', path_version).load_module()
 
 name        = 'vaex'
 author      = 'Maarten A. Breddels'

packages/vaex-contrib/setup.py

@@ -1,10 +1,10 @@
 import os
-import imp
+from importlib.machinery import SourceFileLoader
 from setuptools import setup
 
 dirname = os.path.dirname(__file__)
 path_version = os.path.join(dirname, 'vaex/contrib/_version.py')
-version = imp.load_source('version', path_version)
+version = SourceFileLoader('version', path_version).load_module()
 
 name        = 'vaex'
 license     = 'MIT'

packages/vaex-core/pyproject.toml

@@ -1,8 +1,6 @@
 [build-system]
 # Minimum requirements for the build system to execute.
 requires = [
-    "setuptools",
-    "wheel",
     "oldest-supported-numpy",
     "scikit-build",
     "cmake",