Convert release pipeline to github actions (#50)

* Update cibuildwheel config to build py310 wheels

* Add arm wheel targets

* Convert release pipeline to github actions

* Fix dry run params

* Add a pull_request trigger to CI the release process

* Configure git globally

* Checkout repo in build step

* Add setup python step to verify

* Setup python in build step

* Fix trying to setup qemu on non-linux

* Run shell scripts with bash and error if artifacts are not uploaded

* Call vcvars64 on windows

* Fix syntax

* Fix incorrect comparison

* Quote vcvars64.bat call

* Add a call

* Redo wheel downloading in verify step

* Remove release artifact

* Specify CIBW_ARCHS_*

* Add a find

* Specify to build universal2 wheels on macos

* Fix ubuntu wheel path

* Fix wheel installation on macos

* Specify arch on linux too

* Do [[ instead of [

* Add a sed

* Try to fix macos arm64 build

* Ignore capstone wheel

* Publish repos artifact

* Use https for git clones

* Configure git in release step

* Fix wheel copying

* Format

* Add debug echo

* Invert skip logic

.github/workflows/angr-release.yml

@@ -0,0 +1,214 @@
+name: angr Release
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/angr-release.yml
+      - release-scripts/*
+  schedule:
+    - cron: "0 17 * * 2"
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: "Dry run"
+        default: true
+        type: boolean
+        required: false
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  create:
+    name: Create release
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Configure git
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+      - name: Checkout repos
+        run: release-scripts/checkout_repos.sh
+      - name: Create release commits
+        run: release-scripts/create_release_commits.sh
+      - name: Create sdists
+        run: release-scripts/create_sdist.sh
+      - name: Publish repo artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: repos
+          path: repos
+          if-no-files-found: error
+      - name: Publish sdist artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: sdist
+          path: sdist
+          if-no-files-found: error
+      - name: Check artifacts are valid for PyPI
+        run: |
+          pip install twine
+          twine check sdist/*
+
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-22.04, windows-2022, macos-12, macos-14]
+    needs: create
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up QEMU
+        if: startsWith(matrix.os, 'ubuntu')
+        uses: docker/setup-qemu-action@v3
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Download sdists
+        uses: actions/download-artifact@v4
+        with:
+          name: sdist
+          path: sdist
+      - name: Build wheels
+        if: startsWith(matrix.os, 'windows') != true
+        run: release-scripts/build_wheels.sh sdist
+      - name: Build wheels
+        if: startsWith(matrix.os, 'windows')
+        run: |
+          call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
+          bash release-scripts/build_wheels.sh sdist
+        shell: cmd
+      - name: Upload wheel artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-${{ matrix.os }}
+          path: wheels
+          if-no-files-found: error
+      - name: Check artifacts are valid for PyPI
+        run: |
+          pip install twine
+          twine check sdist/*
+
+  verify:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-22.04, windows-2022, macos-12, macos-14]
+    needs: build
+
+    steps:
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Download wheels artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: wheels-${{ matrix.os }}
+          path: wheels
+
+      - name: Download ubuntu wheels artifact
+        if: startsWith(matrix.os, 'windows') || startsWith(matrix.os, 'macos')
+        uses: actions/download-artifact@v4
+        with:
+          name: wheels-ubuntu-22.04
+          path: wheels-ubuntu
+
+      - name: Find
+        run: find .
+
+      - name: Test wheel install
+        run: |
+          python -m venv angr_venv
+          source angr_venv/bin/activate &> /dev/null || source angr_venv/Scripts/activate
+          export PIP_FIND_LINKS="wheels wheels-ubuntu"
+          if [[ $(uname) == "Darwin" || $(uname) == "Linux" ]]; then
+            pip install --no-binary capstone wheels/angr*$(arch | sed s/i386/x86_64/g).whl
+          else
+            pip install wheels/angr*.whl
+          fi
+
+      - name: Test angr import
+        run: |
+          source angr_venv/bin/activate &> /dev/null || source angr_venv/Scripts/activate
+          python -c "import angr; print('angr imports!')"
+
+  publish:
+    runs-on: ubuntu-22.04
+    needs: verify
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Configure git
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+      # Git release commits
+      - name: Download repos artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: repos
+          path: repos
+
+      - name: Publish release commits
+        run: release-scripts/publish_release_commits.sh
+        env:
+          DRY_RUN: ${{ github.event_name == 'schedule' || github.event.inputs.dry_run == false }}
+          GH_TOKEN: ${{ secrets.GH_TOKEN }}
+
+      - name: Bump versions on master
+        run: release-scripts/bump_versions.sh
+        env:
+          DRY_RUN: ${{ github.event_name == 'schedule' || github.event.inputs.dry_run == false }}
+          GH_TOKEN: ${{ secrets.GH_TOKEN }}
+
+      # PyPI artifacts
+      - name: Create artifacts and dist directories
+        run: mkdir artifacts dist
+
+      - name: Download sdist artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: sdist
+          path: artifacts/sdist
+      - name: Download Ubuntu wheels artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: wheels-ubuntu-22.04
+          path: artifacts/wheels-ubuntu-22.04
+      - name: Download Windows wheels artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: wheels-windows-2022
+          path: artifacts/wheels-windows-2022
+      - name: Download macOS x86_64 wheels artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: wheels-macos-12
+          path: artifacts/wheels-macos-12
+      - name: Download macOS arm64 wheels artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: wheels-macos-14
+          path: artifacts/wheels-macos-14
+
+      - name: Collect all packages to upload
+        run: find artifacts \( -name "*.tar.gz" -o -name "*.whl" \) -exec mv {} dist/ \;
+
+      - name: Publish distribution to PyPI
+        if: (github.event_name == 'schedule' || github.event.inputs.dry_run == false) != true
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          verbose: true

scripts/build_wheels.sh → release-scripts/build_wheels.sh

@@ -9,7 +9,7 @@ function realpath() {
 
 sdist_path="$(realpath "$1")"
 
-python -m pip install build cibuildwheel==2.11.2
+python -m pip install build cibuildwheel==2.18.0
 if [ "$(uname)" == "Linux" ]; then
     pip install auditwheel
 elif [ "$(uname)" == "Darwin" ]; then
@@ -29,7 +29,15 @@ done
 
 export PIP_FIND_LINKS="$sdist_path"
 export CIBW_ENVIRONMENT_LINUX="PIP_FIND_LINKS=/host$PIP_FIND_LINKS"
-export CIBW_BUILD="cp38-manylinux_x86_64 cp38-win_amd64 cp38-macosx_x86_64"
+export CIBW_BUILD="
+    cp310-manylinux_x86_64
+    cp310-manylinux_aarch64
+    cp310-win_amd64
+    cp310-macosx_x86_64
+    cp310-macosx_arm64
+    "
+export CIBW_ARCHS_WINDOWS="AMD64"
+export CIBW_ARCHS_LINUX="x86_64 aarch64"
 export CIBW_REPAIR_WHEEL_COMMAND=""
 for dist in $(ls); do
     package=$(cat $dist/PKG-INFO | grep '^Name: [a-zA-Z0-9-]\+$' | head -n 1 | cut -d' ' -f2)

scripts/bump_versions.sh → release-scripts/bump_versions.sh

@@ -8,7 +8,7 @@ pip install packaging
 
 export CHECKOUT_DIR=$(mktemp -d)
 for r in $REPOS $REPOS_LINUX_ONLY; do
-    git clone git@github.com:angr/$r.git $CHECKOUT_DIR/$r --depth=1 --recursive
+    git clone https://github.com/angr/$r.git $CHECKOUT_DIR/$r --depth=1 --recursive
 done
 
 for i in $(ls $CHECKOUT_DIR); do
@@ -54,7 +54,7 @@ for i in $(ls $CHECKOUT_DIR); do
                 git push origin bump/$VERSION
                 gh pr create \
                     --project angr/$CHECKOUT_DIR \
-                    --assignees "@twizmwazin" \
+                    --assignees "twizmwazin" \
                     --title "Bump version to $VERSION" \
                     --body "Release pipeline failed to automatically push commit" \
                     --head "bump/$VERSION" \

scripts/create_repos.sh → release-scripts/checkout_repos.sh

@@ -6,5 +6,5 @@ mkdir -p "repos"
 echo "!.git" > repos/.artifactignore
 
 for r in $REPOS $REPOS_LINUX_ONLY; do
-    git clone git@github.com:angr/$r.git $CHECKOUT_DIR/$r --depth=1 --recursive
+    git clone https://github.com/angr/$r.git $CHECKOUT_DIR/$r --depth=1 --recursive
 done

scripts/create_sdist.sh → release-scripts/create_sdist.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 set -ex
 
 source $(dirname $0)/vars.sh

scripts/setup_venv_from_artifact.sh → release-scripts/setup_venv_from_artifact.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 set -ex
 
 source $(dirname $0)/vars.sh