DeepFrame Wheels Build #56
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: DeepFrame Wheels Build | |
| permissions: | |
| contents: read | |
| id-token: write | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| build_mac_amd: | |
| description: "Build MacOS x86_64 Wheels" | |
| type: boolean | |
| default: true | |
| build_mac_arm: | |
| description: "Build MacOS ARM Wheels" | |
| type: boolean | |
| default: true | |
| build_linux_amd: | |
| description: "Build Linux x86_64 Wheels" | |
| type: boolean | |
| default: true | |
| build_linux_arm: | |
| description: "Build Linux ARM Wheels" | |
| type: boolean | |
| default: true | |
| runner: | |
| description: Choose MacOS Action Runner Type | |
| type: choice | |
| options: | |
| - "self-hosted" | |
| - "macos-12" | |
| - "macos-13" | |
| - "macos-14" | |
| - "macos-15" | |
| default: "self-hosted" | |
| env: | |
| VCPKG_ROOT: ${{ github.workspace }}/vcpkg | |
| VCPKG_FEATURE_FLAGS: dependencygraph | |
| VCPKG_DISABLE_METRICS: true | |
| BUILD_PRESET: "release" | |
| jobs: | |
| get_macos_matrix: | |
| name: Set MacOS Build Matrix | |
| if: ${{ inputs.build_mac_amd || inputs.build_mac_arm }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: set matrix | |
| id: set-matrix | |
| shell: python | |
| run: |- | |
| import os | |
| import json | |
| runner_types = [] | |
| if "${{ inputs.build_mac_amd }}" == "true": | |
| runner_types.append("large") | |
| if "${{ inputs.build_mac_arm }}" == "true": | |
| runner_types.append("xlarge") | |
| python_to_venv = { | |
| '3.9' : ['9'], | |
| '3.10': ['10'], | |
| '3.11': ['11'], | |
| '3.12': ['12'], | |
| '3.13': ['13'] | |
| } | |
| matrix = [{"python": py, "venv": venv, "type": type} | |
| for py, venvs in python_to_venv.items() | |
| for venv in venvs | |
| for type in runner_types] | |
| with open(os.environ.get("GITHUB_OUTPUT"), "a") as file: | |
| file.write(f"matrix={json.dumps(matrix)}\n") | |
| print(f"[INFO] matrix={json.dumps(matrix)}\n") | |
| build_wheels-macos: | |
| name: MacOS Wheels Build | |
| needs: [get_macos_matrix] | |
| runs-on: ${{ inputs.runner }}-${{ matrix.type }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJSON(needs.get_macos_matrix.outputs.matrix) }} | |
| steps: | |
| - name: set platform name | |
| id: platform-name | |
| shell: bash | |
| env: | |
| TYPE: ${{ matrix.type }} | |
| run: | | |
| case "${TYPE}" in | |
| xlarge) | |
| echo "platform=macosx_11_0_arm64" >> $GITHUB_OUTPUT | |
| echo "arch=arm64" >> $GITHUB_OUTPUT | |
| echo "triplet=arm64-osx" >> $GITHUB_OUTPUT | |
| ;; | |
| large) | |
| echo "platform=macosx_10_9_x86_64" >> $GITHUB_OUTPUT | |
| echo "arch=x86_64" >> $GITHUB_OUTPUT | |
| echo "triplet=x64-osx" >> $GITHUB_OUTPUT | |
| ;; | |
| esac | |
| - name: set vcpkg cache path | |
| id: vcpkg-cache | |
| shell: bash | |
| env: | |
| ARCH: ${{ steps.platform-name.outputs.arch }} | |
| run: | | |
| echo "cache_path=/Users/sre/GitHub/vcpkg-cache-deepframe-${{ steps.platform-name.outputs.triplet }}" >> $GITHUB_OUTPUT | |
| - name: clone repository | |
| uses: actions/checkout@v4.2.2 | |
| - name: setup python ${{ matrix.python }} | |
| uses: actions/setup-python@v5.3.0 | |
| if: inputs.runner != 'self-hosted' | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: setup python ${{ matrix.python }} environment | |
| id: setup | |
| shell: bash | |
| run: |- | |
| python${{ matrix.python }} -m venv "venv_${{ matrix.venv }}" | |
| source "venv_${{ matrix.venv }}/bin/activate" | |
| python"${{ matrix.python }}" -m pip install -qqq -U pip setuptools build cmake==3.29.2 | |
| deactivate | |
| echo -e "\n[bdist_wheel]\nplat-name = ${{ steps.platform-name.outputs.platform }}" >> setup.cfg | |
| - name: install vcpkg | |
| shell: bash | |
| run: git clone https://github.com/microsoft/vcpkg.git && cd vcpkg && git checkout 6f29f12e82a8293156836ad81cc9bf5af41fe836 && ./bootstrap-vcpkg.sh | |
| - name: install vcpkg dependencies | |
| shell: bash | |
| env: | |
| VCPKG_ROOT: ${{ github.workspace }}/vcpkg | |
| VCPKG_TARGET_TRIPLET: ${{ steps.platform-name.outputs.triplet }} | |
| VCPKG_HOST_TRIPLET: ${{ steps.platform-name.outputs.triplet }} | |
| run: | | |
| cd vcpkg | |
| ./vcpkg install --triplet=${{ steps.platform-name.outputs.triplet }} openssl liblzma libiconv zlib | |
| cd .. | |
| - name: clean build artifacts | |
| shell: bash | |
| run: | | |
| # Clean build directory to ensure fresh build | |
| rm -rf build | |
| # Clean architecture-specific FFmpeg installations | |
| rm -rf .ext-* | |
| - name: build py${{ matrix.python }} | |
| env: | |
| CMAKE_OSX_ARCHITECTURES: ${{ steps.platform-name.outputs.arch }} | |
| VCPKG_TARGET_TRIPLET: ${{ steps.platform-name.outputs.triplet }} | |
| VCPKG_HOST_TRIPLET: ${{ steps.platform-name.outputs.triplet }} | |
| VCPKG_BINARY_SOURCES: "clear" | |
| VCPKG_DISABLE_METRICS: true | |
| VCPKG_OVERLAY_TRIPLETS: "" | |
| VCPKG_OVERLAY_PORTS: "" | |
| shell: bash | |
| run: |- | |
| CMAKE_PATH=$(whereis cmake | awk '{print $2}') | |
| CMAKE_DIR=$(dirname "$CMAKE_PATH") | |
| echo "$CMAKE_DIR" >> $GITHUB_PATH | |
| source "venv_${{ matrix.venv }}/bin/activate" | |
| python"${{ matrix.python }}" -m pip install -r requirements-dev.txt && | |
| python"${{ matrix.python }}" scripts/build.py "${{ env.BUILD_PRESET }}" | |
| python"${{ matrix.python }}" -m build | |
| deactivate | |
| - name: run tests | |
| if: matrix.type != 'x86_64' | |
| shell: bash | |
| run: |- | |
| source "venv_${{ matrix.venv }}/bin/activate" | |
| timeout -k 10s 30m python"${{ matrix.python }}" -m pytest | |
| deactivate | |
| - uses: actions/upload-artifact@v4.6.1 | |
| if: ${{ always() }} | |
| with: | |
| name: "deepframe-macos-py${{ matrix.python }}-${{ matrix.type }}-wheels" | |
| path: "dist/*.whl" | |
| get_linux_matrix: | |
| name: Set Linux Build Matrix | |
| if: ${{ inputs.build_linux_arm || inputs.build_linux_amd }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: set matrix | |
| id: set-matrix | |
| shell: python | |
| run: |- | |
| import os | |
| import json | |
| runner_types = [] | |
| if "${{ inputs.build_linux_amd }}" == "true": | |
| runner_types.append("x86_64") | |
| if "${{ inputs.build_linux_arm }}" == "true": | |
| runner_types.append("aarch64") | |
| python_to_venv = { | |
| '3.9' : ['9'], | |
| '3.10': ['10'], | |
| '3.11': ['11'], | |
| '3.12': ['12'], | |
| '3.13': ['13'] | |
| } | |
| matrix = [{"python": py, "venv": venv, "type": type} | |
| for py, venvs in python_to_venv.items() | |
| for venv in venvs | |
| for type in runner_types] | |
| with open(os.environ.get("GITHUB_OUTPUT"), "a") as file: | |
| file.write(f"matrix={json.dumps(matrix)}\n") | |
| print(f"[INFO] matrix={json.dumps(matrix)}\n") | |
| build_wheels_linux: | |
| name: Linux Wheels Build | |
| needs: [get_linux_matrix] | |
| runs-on: ${{ matrix.type == 'x86_64' && 'ubuntu-24.04' || 'ubuntu-24.04-arm' }} | |
| container: us-central1-docker.pkg.dev/activeloop-ops/operations/pypa-manylinux2014_${{ matrix.type }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJSON(needs.get_linux_matrix.outputs.matrix) }} | |
| env: | |
| VCPKG_BINARY_SOURCES: "clear;files,/vcpkg-cache,readwrite" | |
| steps: | |
| - name: clone repository | |
| shell: bash | |
| run: git clone -b "${{ github.ref_name }}" "https://activeloop-bot:${{ secrets.ORG_GH_BOT_PAT }}@github.com/${{ github.repository }}" . | |
| - name: setup environment | |
| shell: bash | |
| run: | | |
| echo -e "\n[bdist_wheel]\nplat-name = manylinux2014_${{ matrix.type }}" >> setup.cfg | |
| - name: download vcpkg cache | |
| shell: bash | |
| run: source scripts/build_scripts/get_creds.sh && python3.11 scripts/build_scripts/manage_cache.py download deepframe | |
| - name: install vcpkg | |
| shell: bash | |
| run: git clone https://github.com/microsoft/vcpkg.git && cd vcpkg && git checkout 6f29f12e82a8293156836ad81cc9bf5af41fe836 && ./bootstrap-vcpkg.sh | |
| - name: build py${{ matrix.python }} | |
| shell: bash | |
| run: |- | |
| python"${{ matrix.python }}" -m pip install -r requirements-dev.txt && | |
| python"${{ matrix.python }}" scripts/build.py "${{ env.BUILD_PRESET }}" | |
| python"${{ matrix.python }}" -m build | |
| - name: save vcpkg cache | |
| if: ${{ matrix.python == '3.11' }} | |
| shell: bash | |
| run: source scripts/build_scripts/get_creds.sh && python3.11 scripts/build_scripts/manage_cache.py upload deepframe | |
| - name: run tests | |
| shell: bash | |
| run: timeout -k 10s 30m python"${{ matrix.python }}" -m pytest | |
| - name: upload wheels to s3 | |
| if: ${{ always() }} | |
| env: | |
| RUN_ID: ${{ github.run_id }} | |
| TYPE: ${{ matrix.type }} | |
| PYTHON: ${{ matrix.python }} | |
| shell: bash | |
| run: source scripts/build_scripts/get_creds.sh && python3.11 scripts/build_scripts/upload_wheels.py deepframe |