Merge pull request #89 from paulsengroup/ci/optimize

Optimize CI

.github/workflows/pip.yml

@@ -31,7 +31,8 @@ on:
       - "pyproject.toml"
 
 env:
-  CONAN_HOME: "${{ github.workspace }}/conan/"
+  CONAN_HOME: "${{ github.workspace }}/.conan2/"
+  HICTK_CI: "1"
 
 # https://stackoverflow.com/a/72408109
 concurrency:
@@ -47,17 +48,16 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [windows-2019, macos-latest, ubuntu-latest]
+        os: [windows-2019, macos-latest, ubuntu-latest]
         python-version: ["3.9", "3.12"]
 
-    runs-on: ${{ matrix.platform }}
+    runs-on: ${{ matrix.os }}
 
     steps:
       - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
 
-      - uses: actions/setup-python@v5
+      - name: Setup Python
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -69,17 +69,17 @@ jobs:
         run: |
           hash="${{ hashFiles('conanfile.py') }}"
 
-            echo "conan-key=pip-$hash" >> $GITHUB_OUTPUT
+          echo "conan-key=pip-${{ matrix.os }}-$hash" >> $GITHUB_OUTPUT
 
       - name: Restore Conan cache
         id: cache-conan
         uses: actions/cache/restore@v4
         with:
           key: conan-${{ steps.cache-key.outputs.conan-key }}
-          path: ${{ env.CONAN_HOME }}
+          path: ${{ env.CONAN_HOME }}/p
 
       - name: Override default compilers (macOS)
-        if: matrix.platform == 'macos-latest'
+        if: matrix.os == 'macos-latest'
         run: |
           echo 'CC=clang' >> $GITHUB_ENV
           echo 'CXX=clang++' >> $GITHUB_ENV
@@ -92,7 +92,7 @@ jobs:
         if: steps.cache-conan.outputs.cache-hit != 'true'
         with:
           key: conan-${{ steps.cache-key.outputs.conan-key }}
-          path: ${{ env.CONAN_HOME }}
+          path: ${{ env.CONAN_HOME }}/p
 
       - name: Test
         run: python -m pytest test -v

.github/workflows/wheels.yml

@@ -1,14 +1,15 @@
 # Copyright (C) 2023 Roberto Rossini ([email protected])
 # SPDX-License-Identifier: MIT
 
-name: Wheels
+name: Build wheels
 
 on:
   workflow_dispatch:
   push:
     branches: [main]
     paths:
-      - ".github/workflows/pip.yml"
+      - ".github/workflows/build-conan-deps.yml"
+      - ".github/workflows/wheels.yml"
       - "cmake/**"
       - "src/**"
       - "test/**"
@@ -22,7 +23,8 @@ on:
 
   pull_request:
     paths:
-      - ".github/workflows/pip.yml"
+      - ".github/workflows/build-conan-deps.yml"
+      - ".github/workflows/wheels.yml"
       - "cmake/**"
       - "src/**"
       - "test/**"
@@ -45,18 +47,103 @@ defaults:
   run:
     shell: bash
 
-env:
-  CONAN_HOME: "${{ github.workspace }}/conan/"
-  CCACHE_DIR: "${{ github.workspace }}/ccache/"
-
 jobs:
+  matrix-factory:
+    name: Generate job matrix
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-result.outputs.matrix }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install cibuildwheel
+        run: pip install 'cibuildwheel>=2.21'
+
+      - name: Generate matrix
+        id: set-result
+        run: |
+          CIBW_ARCHS_LINUX=x86_64 cibuildwheel \
+            --print-build-identifiers \
+            --platform linux |
+          jq -nRc '{"wheel-config": inputs, "os": "ubuntu-latest", "arch": "x86_64"}' |
+          tee configs.json
+
+          if [ '${{ github.event_name }}' != 'pull_request' ]; then
+            CIBW_ARCHS_LINUX=aarch64 cibuildwheel \
+              --print-build-identifiers \
+              --platform linux |
+            jq -nRc '{"wheel-config": inputs, "os": "ubuntu-latest", "arch": "aarch64"}' |
+            tee -a configs.json
+          fi
+
+          CIBW_ARCHS_MACOS=x86_64 cibuildwheel \
+            --print-build-identifiers \
+            --platform macos |
+          jq -nRc '{"wheel-config": inputs, "os": "macos-13", "arch": "x86_64"}' |
+          tee -a configs.json
+
+          CIBW_ARCHS_MACOS=arm64 cibuildwheel \
+            --print-build-identifiers \
+            --platform macos |
+          jq -nRc '{"wheel-config": inputs, "os": "macos-14", "arch": "arm64"}' |
+          tee -a configs.json
+
+          CIBW_ARCHS_WINDOWS=AMD64 cibuildwheel \
+            --print-build-identifiers \
+            --platform windows |
+          jq -nRc '{"wheel-config": inputs, "os": "windows-2019", "arch": "AMD64"}' |
+          tee -a configs.json
+
+          MATRIX="$(jq -sc < configs.json)"
+
+          echo "matrix={\"include\": $MATRIX }" | tee -a "$GITHUB_OUTPUT"
+
+  build-conan-deps-linux-x86:
+    name: Build Conan deps (Linux x86_64)
+    uses: paulsengroup/hictkpy/.github/workflows/build-conan-deps.yml@main
+    with:
+      arch: x86_64
+      os: manylinux_2_28
+
+  build-conan-deps-linux-arm64:
+    name: Build Conan deps (Linux arm64)
+    uses: paulsengroup/hictkpy/.github/workflows/build-conan-deps.yml@main
+    with:
+      arch: aarch64
+      os: manylinux_2_28
+
+  build-conan-deps-macos-x86:
+    name: Build Conan deps (macOS x86_64)
+    uses: paulsengroup/hictkpy/.github/workflows/build-conan-deps.yml@main
+    with:
+      arch: x86_64
+      os: macos-13
+
+  build-conan-deps-macos-arm64:
+    name: Build Conan deps (macOS arm64)
+    uses: paulsengroup/hictkpy/.github/workflows/build-conan-deps.yml@main
+    with:
+      arch: arm64
+      os: macos-14
+
+  build-conan-deps-windows-x86:
+    name: Build Conan deps (Windows x86_64)
+    uses: paulsengroup/hictkpy/.github/workflows/build-conan-deps.yml@main
+    with:
+      arch: AMD64
+      os: windows-2019
+
   build-sdist:
     name: Build SDist
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
 
       - name: Build SDist
         run: pipx run build --sdist
@@ -71,50 +158,110 @@ jobs:
 
   build-wheels:
     name: Build Wheels
+    needs:
+      - matrix-factory
+      - build-conan-deps-linux-x86
+      - build-conan-deps-linux-arm64
+      - build-conan-deps-macos-x86
+      - build-conan-deps-macos-arm64
+      - build-conan-deps-windows-x86
     runs-on: ${{ matrix.os }}
 
     strategy:
-      matrix:
-        os: [ubuntu-latest, macos-13, macos-14, windows-2019]
       fail-fast: false
+      matrix: ${{ fromJson(needs.matrix-factory.outputs.matrix) }}
+
+    env:
+      CONAN_HOME: "${{ github.workspace }}/.conan2"
+      HICTKPY_CONAN_INSTALL_ARGS: "--settings=compiler.cppstd=17;--settings=build_type=Release;--build=never;--options=*/*:shared=False"
 
     steps:
       - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
 
       - name: Set up QEMU
-        if: matrix.os == 'ubuntu-latest'
+        if: startsWith(matrix.os, 'ubuntu')
         uses: docker/setup-qemu-action@v3
         with:
-          platforms: all
+          platforms: linux/amd64,linux/arm64
 
-      - name: Build wheels (PR)
-        uses: pypa/[email protected]
-        if: github.event_name == 'pull_request'
-        env:
-          CIBW_ARCHS_LINUX: "x86_64"
-          CIBW_ARCHS_WINDOWS: "AMD64"
-          CIBW_ENVIRONMENT: "PIP_VERBOSE=1"
-          CIBW_ENVIRONMENT_MACOS: "CC=clang CXX=clang++ MACOSX_DEPLOYMENT_TARGET=10.15"
+      - name: Restore Conan cache (Linux; x86_64)
+        if: startsWith(matrix.os, 'ubuntu') && matrix.arch == 'x86_64'
+        uses: actions/cache/restore@v4
+        with:
+          key: ${{ needs.build-conan-deps-linux-x86.outputs.conan-key }}
+          path: ${{ env.CONAN_HOME }}/p
+          fail-on-cache-miss: true
+
+      - name: Restore Conan cache (Linux; aarch64)
+        if: |
+          github.event_name != 'pull_request' &&
+          startsWith(matrix.os, 'ubuntu') &&
+          matrix.arch == 'aarch64'
+        uses: actions/cache/restore@v4
+        with:
+          key: ${{ needs.build-conan-deps-linux-arm64.outputs.conan-key }}
+          path: ${{ env.CONAN_HOME }}/p
+          fail-on-cache-miss: true
+
+      - name: Restore Conan cache (macOS; x86_64)
+        if: startsWith(matrix.os, 'macos') && matrix.arch == 'x86_64'
+        uses: actions/cache/restore@v4
+        with:
+          key: ${{ needs.build-conan-deps-macos-x86.outputs.conan-key }}
+          path: ${{ env.CONAN_HOME }}/p
+          fail-on-cache-miss: true
+
+      - name: Restore Conan cache (macOS; aarch64)
+        if: startsWith(matrix.os, 'macos') && matrix.arch == 'arm64'
+        uses: actions/cache/restore@v4
+        with:
+          key: ${{ needs.build-conan-deps-macos-arm64.outputs.conan-key }}
+          path: ${{ env.CONAN_HOME }}/p
+          fail-on-cache-miss: true
+
+      - name: Restore Conan cache (Windows; AMD64)
+        if: startsWith(matrix.os, 'windows') && matrix.arch == 'AMD64'
+        uses: actions/cache/restore@v4
+        with:
+          key: ${{ needs.build-conan-deps-windows-x86.outputs.conan-key }}
+          path: ${{ needs.build-conan-deps-windows-x86.outputs.conan-home }}\p
+          fail-on-cache-miss: true
 
       - name: Build wheels
         uses: pypa/[email protected]
-        if: github.event_name != 'pull_request'
+        with:
+          only: ${{ matrix.wheel-config }}
         env:
-          CIBW_ARCHS_LINUX: "x86_64 aarch64"
-          CIBW_ARCHS_WINDOWS: "AMD64"
-          CIBW_ENVIRONMENT: "PIP_VERBOSE=1"
-          CIBW_ENVIRONMENT_MACOS: "CC=clang CXX=clang++ MACOSX_DEPLOYMENT_TARGET=10.15"
+          CIBW_ENVIRONMENT_LINUX: >
+            PIP_VERBOSE=1
+            HICTK_CI=1
+            CONAN_HOME='/host${{ env.CONAN_HOME }}'
+          CIBW_ENVIRONMENT_MACOS: >
+            CC=clang
+            CXX=clang++
+            PIP_VERBOSE=1
+            HICTK_CI=1
+            MACOSX_DEPLOYMENT_TARGET=10.15
+            HICTKPY_CONAN_INSTALL_ARGS="${{ env.HICTKPY_CONAN_INSTALL_ARGS }};-s:h os.version=${{ env.MACOSX_DEPLOYMENT_TARGET }}"
+          CIBW_ENVIRONMENT_WINDOWS: >
+            PIP_VERBOSE=1
+            HICTK_CI=1
+          CIBW_ENVIRONMENT_PASS_LINUX: >
+            CONAN_HOME
+            HICTKPY_CONAN_INSTALL_ARGS
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28
+          CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
 
       - name: Verify clean directory
         run: git diff --exit-code
 
       - name: Upload wheels
         uses: actions/upload-artifact@v4
         with:
-          name: "wheels-${{ matrix.os }}"
+          name: "wheels-${{ matrix.wheel-config }}"
           path: wheelhouse/*.whl
+          if-no-files-found: error
+          retention-days: 1
 
   package-artifacts:
     name: Package artifacts
@@ -142,3 +289,22 @@ jobs:
         with:
           name: dist
           path: dist.tar
+          if-no-files-found: error
+          retention-days: 30
+
+  build-wheels-status-check:
+    name: Status Check (Build wheels)
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs:
+      - build-sdist
+      - build-wheels
+      - package-artifacts
+
+    steps:
+      - name: Collect job results
+        if: |
+          needs.build-sdist.result != 'success'  ||
+          needs.build-wheels.result != 'success' ||
+          needs.package-artifacts.result != 'success'
+        run: exit 1

pyproject.toml

@@ -52,7 +52,7 @@ cmake.build-type = "Release"
 BUILD_SHARED_LIBS = "OFF"
 CMAKE_OSX_DEPLOYMENT_TARGET = "10.15"
 CMAKE_PROJECT_TOP_LEVEL_INCLUDES = { env="HICTKPY_PROJECT_TOP_LEVEL_INCLUDES", default="cmake/conan_provider.cmake" }
-CONAN_INSTALL_ARGS = "--settings=compiler.cppstd=17;--settings=build_type=Release;--build=missing;--build=b2/*;--update;--options=*/*:shared=False"
+CONAN_INSTALL_ARGS = { env="HICTKPY_CONAN_INSTALL_ARGS", default="--settings=compiler.cppstd=17;--settings=build_type=Release;--build=missing;--update;--options=*/*:shared=False" }
 HICTK_BUILD_BENCHMARKS = "OFF"
 HICTK_BUILD_EXAMPLES = "OFF"
 HICTK_BUILD_TOOLS = "OFF"