diff --git a/.github/workflows/build_packages.yml b/.github/workflows/build_packages.yml new file mode 100644 index 000000000..ed610537b --- /dev/null +++ b/.github/workflows/build_packages.yml @@ -0,0 +1,55 @@ +# Copyright 2024 The IREE Authors +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: Build packages + +on: + workflow_dispatch: + schedule: + # Runs at 11:00 AM UTC, which is 3:00 AM PST (UTC-8) + - cron: '0 11 * * *' + +jobs: + build_packages: + if: ${{ github.repository_owner == 'iree-org' || github.event_name != 'schedule' }} + runs-on: ubuntu-22.04 + permissions: + contents: write + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: "Setting up Python" + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: 3.11 + + - name: Install dependencies + run: pip install -r ./build_tools/requirements-packaging.txt + + - name: Build iree-turbine release candidate + run: | + ./build_tools/compute_local_version.py -rc --write-json + ./build_tools/build_release.py --no-download + + - name: Upload python wheels + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + if-no-files-found: error + name: snapshot + path: wheelhouse + + - name: Release python wheels + uses: ncipollo/release-action@2c591bcc8ecdcd2db72b97d6147f871fcd833ba5 # v1.14.0 + with: + artifacts: wheelhouse/*.whl + tag: "dev-wheels" + name: "dev-wheels" + body: "Automatic snapshot release of iree-turbine python wheels." + removeArtifacts: false + allowUpdates: true + replacesArtifacts: true + makeLatest: false diff --git a/.github/workflows/test_build_release.yml b/.github/workflows/test_build_release.yml index 696820ddc..458a71e27 100644 --- a/.github/workflows/test_build_release.yml +++ b/.github/workflows/test_build_release.yml @@ -4,7 +4,7 @@ # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -name: Build Release +name: Test Build Release on: workflow_dispatch: @@ -37,10 +37,15 @@ jobs: id: setup_python uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: - python-version: ${{matrix.version}} + python-version: ${{ matrix.version }} + + - name: Install dependencies + run: pip install -r ./build_tools/requirements-packaging.txt - name: Build Release Wheels - run: ./build_tools/build_release.py --package-version 2.5.0.dev + run: | + ./build_tools/compute_local_version.py -dev --write-json + ./build_tools/build_release.py - name: Validate Release Build if: ${{ !cancelled() }} diff --git a/.gitignore b/.gitignore index 3f5c75efc..d1cc189e5 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,10 @@ wheelhouse *.whl *.venv -#Model artifacts +# Local-only config options +version_local.json + +# Model artifacts *.pt *.safetensors *.gguf diff --git a/build_tools/build_release.py b/build_tools/build_release.py index 408fcdd91..567b3973e 100755 --- a/build_tools/build_release.py +++ b/build_tools/build_release.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # Copyright 2024 Advanced Micro Devices, Inc. +# Copyright 2024 The IREE Authors # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. @@ -10,17 +11,15 @@ See docs/releasing.md for usage. """ +from pathlib import Path import argparse -from datetime import date -import json import os -from pathlib import Path import shlex import subprocess +import sys REPO_ROOT = Path(__file__).resolve().parent.parent -VERSION_INFO_FILE = REPO_ROOT / "version_info.json" WHEEL_DIR = REPO_ROOT / "wheelhouse" # The platform flags that we will download IREE wheels for. This must match @@ -54,30 +53,6 @@ ] -def eval_version(version_spec: str): - date_stamp = date.today().strftime("%Y%m%d") - return version_spec.replace("YYYYMMDD", date_stamp) - - -def write_version_info(args): - with open(VERSION_INFO_FILE, "rt") as f: - info_dict = json.load(f) - - # Compute package-version. - package_version = eval_version(args.package_version) - if args.package_pre_version: - package_version += eval_version(args.package_pre_version) - if args.package_post_version: - package_version += f".{eval_version(args.package_post_version)}" - info_dict["package-version"] = package_version - - with open(VERSION_INFO_FILE, "wt") as f: - json.dump(info_dict, f, indent=2) - f.write("\n") - - print(f"Updated version_info.json:\n{json.dumps(info_dict, indent=2)}") - - def exec(args, env=None): args = [str(s) for s in args] print(f": Exec: {shlex.join(args)}") @@ -91,6 +66,8 @@ def exec(args, env=None): def download_requirements(requirements_file, platforms=()): args = [ + sys.executable, + "-m", "pip", "download", "-d", @@ -113,6 +90,8 @@ def download_iree_binaries(): for platform_args in IREE_PLATFORM_ARGS: print("Downloading for platform:", platform_args) args = [ + sys.executable, + "-m", "pip", "download", "-d", @@ -135,31 +114,33 @@ def download_iree_binaries(): exec(args) -def build_wheel(path, env=None): - exec( - ["pip", "wheel", "--no-index", "-f", WHEEL_DIR, "-w", WHEEL_DIR, path], env=env - ) +def build_wheel(args, path): + build_args = [ + sys.executable, + "-m", + "pip", + "wheel", + "-f", + WHEEL_DIR, + "-w", + WHEEL_DIR, + path, + ] + if args.no_download: + build_args.extend(["--disable-pip-version-check", "--no-deps"]) + else: + build_args.extend(["--no-index"]) + + exec(build_args, env=None) def main(): parser = argparse.ArgumentParser() - parser.add_argument("--package-version", help="Version to use", required=True) - parser.add_argument( - "--package-pre-version", - help="Pre-release version segment or (YYYYMMDD)", - default="", - ) - parser.add_argument( - "--package-post-version", - help="Post-release version segment or (YYYYMMDD)", - default="", - ) parser.add_argument( "--no-download", help="Disable dep download", action="store_true" ) args = parser.parse_args() - write_version_info(args) WHEEL_DIR.mkdir(parents=True, exist_ok=True) if not args.no_download: @@ -171,7 +152,7 @@ def main(): download_requirements(REPO_ROOT / "requirements.txt") print("Building iree-turbine") - build_wheel(REPO_ROOT) + build_wheel(args, REPO_ROOT) if __name__ == "__main__": diff --git a/build_tools/compute_local_version.py b/build_tools/compute_local_version.py new file mode 100755 index 000000000..a02dba86a --- /dev/null +++ b/build_tools/compute_local_version.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# Copyright 2024 The IREE Authors +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# This scripts grabs the X.Y.Z[.dev]` version identifier from `version.json` +# and writes the corresponding `X.Y.ZrcYYYYMMDD` version identifier to +# `version_local.json`. + +from datetime import datetime +from packaging.version import Version +from pathlib import Path +import argparse +import json +import subprocess + + +parser = argparse.ArgumentParser() +parser.add_argument("--write-json", action="store_true") + +release_type = parser.add_mutually_exclusive_group(required=True) +release_type.add_argument("-stable", "--stable-release", action="store_true") +release_type.add_argument("-rc", "--nightly-release", action="store_true") +release_type.add_argument("-dev", "--development-release", action="store_true") +release_type.add_argument("--custom-string", action="store", type=str) + +args = parser.parse_args() + +REPO_ROOT = Path(__file__).parent.parent +VERSION_FILE_PATH = REPO_ROOT / "version.json" +VERSION_LOCAL_FILE_PATH = REPO_ROOT / "version_local.json" + + +def load_version_from_file(version_file): + with open(version_file, "rt") as f: + return json.load(f) + + +def write_version_to_file(version_file, version): + with open(version_file, "w") as f: + json.dump({"package-version": version}, f, indent=2) + f.write("\n") + + +version_info = load_version_from_file(VERSION_FILE_PATH) +package_version = version_info.get("package-version") +current_version = Version(package_version).base_version + +if args.nightly_release: + current_version += "rc" + datetime.today().strftime("%Y%m%d") +elif args.development_release: + current_version += ( + ".dev+" + + subprocess.check_output(["git", "rev-parse", "HEAD"]).decode("ascii").strip() + ) +elif args.custom_string: + current_version += args.custom_string + +if args.write_json: + write_version_to_file(VERSION_LOCAL_FILE_PATH, current_version) + +print(current_version) diff --git a/build_tools/promote_whl_from_rc_to_final.py b/build_tools/promote_whl_from_rc_to_final.py new file mode 100755 index 000000000..f7b1a93af --- /dev/null +++ b/build_tools/promote_whl_from_rc_to_final.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# Copyright 2024 The IREE Authors +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# This scripts takes a file like +# 'iree_turbine-3.1.0rc20241204-py3-none-any.whl' +# with embedded version '3.1.0rc20241204' as input and then drops the +# 'rcYYYYMMDD' suffix from both the embedded version and file name. +# +# Typical usage: +# pip install -r requirements-packaging.txt +# ./promote_whl_from_rc_to_final.py /path/to/file.whl --delete-old-wheel + +import argparse +from change_wheel_version import change_wheel_version +from packaging.version import Version +from pathlib import Path +from pkginfo import Wheel + + +def parse_arguments(): + parser = argparse.ArgumentParser() + parser.add_argument( + "input_file", + help="Path to the input .whl file to promote", + type=Path, + ) + parser.add_argument( + "--delete-old-wheel", + help="Deletes the original wheel after successfully promoting it", + action="store_true", + default=False, + ) + return parser.parse_args() + + +def main(args): + original_wheel_path = args.input_file + print(f"Promoting whl from rc to final: '{original_wheel_path}'") + + original_wheel = Wheel(original_wheel_path) + original_version = Version(original_wheel.version) + base_version = original_version.base_version + print( + f" Original wheel version is '{original_version}' with base '{base_version}'" + ) + + if str(base_version) == str(original_version): + print(" Version is already a release version, skipping") + return + + print(f" Changing to base version: '{base_version}'") + new_wheel_path = change_wheel_version(original_wheel_path, str(base_version), None) + print(f" New wheel path is '{new_wheel_path}'") + + new_wheel = Wheel(new_wheel_path) + new_version = Version(new_wheel.version) + print(f" New wheel version is '{new_version}'") + + if args.delete_old_wheel: + print(" Deleting original wheel") + original_wheel_path.unlink() + + +if __name__ == "__main__": + main(parse_arguments()) diff --git a/build_tools/pypi_deploy.sh b/build_tools/pypi_deploy.sh new file mode 100755 index 000000000..7c29063b5 --- /dev/null +++ b/build_tools/pypi_deploy.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +# Copyright 2024 The IREE Authors +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# This script promotes Python packages from nightly releases to PyPI. +# +# Prerequisites: +# * For deploying to PyPI, you will need to have credentials set up. See +# https://packaging.python.org/en/latest/tutorials/packaging-projects/#uploading-the-distribution-archives +# * Install requirements, e.g. in a Python virtual environment (venv): +# `pip install -r requirements-packaging.txt` +# * Choose a release candidate to promote from +# https://github.com/iree-org/iree-turbine/releases/tag/dev-wheels +# +# Usage: +# python -m venv .venv +# source .venv/bin/activate +# pip install -r ./requirements-packaging.txt +# ./pypi_deploy.sh 3.1.0rc20241204 + +set -euo pipefail + +RELEASE="$1" + +SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")"; +TMPDIR="$(mktemp --directory --tmpdir iree_turbine_pypi_wheels.XXXXX)" +ASSETS_PAGE="https://github.com/iree-org/iree-turbine/releases/expanded_assets/dev-wheels" + +function download_wheels() { + echo "" + echo "Downloading wheels for '${RELEASE}'..." + + python -m pip download iree-turbine==${RELEASE} --no-deps -f ${ASSETS_PAGE} + + echo "" + echo "Downloaded wheels:" + ls +} + +function edit_release_versions() { + echo "" + echo "Editing release versions..." + for file in * + do + ${SCRIPT_DIR}/promote_whl_from_rc_to_final.py ${file} --delete-old-wheel + done + + echo "Edited wheels:" + ls +} + +function upload_wheels() { + # TODO: list packages that would be uploaded, pause, prompt to continue + echo "" + echo "Uploading wheels:" + ls + twine upload --verbose * +} + +function main() { + echo "Changing into ${TMPDIR}" + cd "${TMPDIR}" + # TODO: check_requirements (using pip) + + download_wheels + edit_release_versions + upload_wheels +} + +main diff --git a/build_tools/requirements-packaging.txt b/build_tools/requirements-packaging.txt new file mode 100644 index 000000000..dcc32d47a --- /dev/null +++ b/build_tools/requirements-packaging.txt @@ -0,0 +1,4 @@ +change_wheel_version +packaging +pkginfo +twine diff --git a/docs/releasing.md b/docs/releasing.md index f56b04fdd..a2e6c3323 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -6,31 +6,69 @@ https://pypi.org/project/iree-base-runtime/ packages. Releases can either be conducted independently, or they can be coordinated across projects by initiating a release here. -## Start with a clean test directory +Promoting a nightly release allows for a consistent process across multiple +projects while building locally lets release engineers take more explicit +control over the process, while risking human error. + +## Promoting a nightly release + +To promote a nightly release: + +```bash +cd build_tools/ +python -m venv .venv +source .venv/bin/activate +pip install -r ./requirements-packaging.txt + +# NOTE: choose the nightly version to promote here! +./pypi_deploy.sh 3.1.0rc20241204 +``` + +## Building locally + +To build locally: + +### Start with a clean test directory ```bash rm -rf wheelhouse/ ``` -## Building Artifacts +### Building Artifacts -Build a pre-release: +Build a dev release (e.g. `3.1.0.dev+6879a433eecc1e0b2cdf6c6dbcad901c77d97ac8`): ```bash -./build_tools/build_release.py --package-version 2.5.0 --package-pre-version=rcYYYYMMDD +python ./build_tools/compute_local_version.py -dev --write-json +python ./build_tools/build_release.py ``` -Build an official release: +Build a release candidate (e.g. `3.1.0rc20241204`): ```bash -./build_tools/build_release.py --package-version 2.5.0 +python ./build_tools/compute_local_version.py -rc --write-json +python ./build_tools/build_release.py +``` + +Build an official release (e.g. `3.1.0`): + +```bash +python ./build_tools/compute_local_version.py -stable --write-json +python ./build_tools/build_release.py ``` This will download all deps, including wheels for all supported platforms and Python versions for iree-base-compiler and iree-base-runtime. All wheels will be placed in the `wheelhouse/` directory. -## Testing +If you just want to build without downloading wheels, run + +```bash +python ./build_tools/build_release.py --no-download +# Note that the test scripts referenced below won't work with this. +``` + +### Testing ```bash ./build_tools/post_build_release_test.sh @@ -42,7 +80,7 @@ This will 2. Install wheels from the `wheelhouse/` directory 3. Run `pytest` tests -## Push +### Push From the testing venv, verify that everything is sane: @@ -62,7 +100,7 @@ Push built wheels: twine upload wheelhouse/iree_turbine-* ``` -## Install from PyPI and Sanity Check +### Install from PyPI and Sanity Check From the testing venv: diff --git a/setup.py b/setup.py index 0488f85b3..6861d2306 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ # Copyright 2023 Advanced Micro Devices, Inc. +# Copyright 2024 The IREE Authors # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. @@ -12,28 +13,28 @@ from setuptools import find_namespace_packages, setup THIS_DIR = os.path.realpath(os.path.dirname(__file__)) -REPO_DIR = THIS_DIR -VERSION_INFO_FILE = os.path.join(REPO_DIR, "version_info.json") - -TURBINE_PACKAGE_NAME = "iree-turbine" - -with open( - os.path.join( - REPO_DIR, - "README.md", - ), - "rt", -) as f: - README = f.read() +REPO_ROOT = THIS_DIR + +VERSION_FILE = os.path.join(REPO_ROOT, "version.json") +VERSION_FILE_LOCAL = os.path.join(REPO_ROOT, "version_local.json") -def load_version_info(): - with open(VERSION_INFO_FILE, "rt") as f: +def load_version_info(version_file): + with open(version_file, "rt") as f: return json.load(f) -version_info = load_version_info() +try: + version_info = load_version_info(VERSION_FILE_LOCAL) +except FileNotFoundError: + print("version_local.json not found. Default to dev build") + version_info = load_version_info(VERSION_FILE) + PACKAGE_VERSION = version_info["package-version"] +print(f"Using PACKAGE_VERSION: '{PACKAGE_VERSION}'") + +with open(os.path.join(REPO_ROOT, "README.md"), "rt") as f: + README = f.read() packages = find_namespace_packages( include=[ @@ -78,7 +79,7 @@ def initialize_options(self): setup( - name=f"{TURBINE_PACKAGE_NAME}", + name="iree-turbine", version=f"{PACKAGE_VERSION}", author="IREE Authors", author_email="iree-technical-discussion@lists.lfaidata.foundation", diff --git a/version_info.json b/version.json similarity index 100% rename from version_info.json rename to version.json