From 6814b51d71d8978176c1ebe0f9b48d633a0b33e3 Mon Sep 17 00:00:00 2001 From: David Elner Date: Wed, 15 Jul 2026 21:58:07 +0000 Subject: [PATCH] Changed: Use shared release workflow for JS+PY packages --- .github/workflows/publish-js.yaml | 206 ---------------- .github/workflows/publish-py.yaml | 233 ------------------ .github/workflows/publish.yaml | 377 ++++++++++++++++++++++++++---- docs/PUBLISHING.md | 226 +++--------------- 4 files changed, 363 insertions(+), 679 deletions(-) delete mode 100644 .github/workflows/publish-js.yaml delete mode 100644 .github/workflows/publish-py.yaml diff --git a/.github/workflows/publish-js.yaml b/.github/workflows/publish-js.yaml deleted file mode 100644 index 57581709..00000000 --- a/.github/workflows/publish-js.yaml +++ /dev/null @@ -1,206 +0,0 @@ -name: publish-js - -concurrency: - group: publish-js-${{ inputs.release_type }}-${{ inputs.branch }} - cancel-in-progress: false - -on: - workflow_dispatch: - inputs: - release_type: - description: Release type - required: true - default: stable - type: choice - options: - - stable - - prerelease - branch: - description: Branch to release from - required: true - default: main - type: string - prerelease_suffix: - description: Optional shared prerelease suffix - required: false - default: "" - type: string - -jobs: - prepare-release: - runs-on: ubuntu-latest - timeout-minutes: 10 - outputs: - version: ${{ steps.release_metadata.outputs.version }} - release_tag: ${{ steps.release_metadata.outputs.release_tag }} - branch: ${{ steps.release_metadata.outputs.branch }} - commit: ${{ steps.release_metadata.outputs.commit }} - release_type: ${{ steps.release_metadata.outputs.release_type }} - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - fetch-depth: 1 - ref: ${{ inputs.branch }} - - name: Check version sync - run: python3 .github/scripts/check_version_sync.py - - name: Set up Node.js - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 - with: - node-version-file: .tool-versions - - name: Determine release metadata - id: release_metadata - env: - RELEASE_TYPE: ${{ inputs.release_type }} - TARGET_BRANCH: ${{ inputs.branch }} - PRERELEASE_SUFFIX: ${{ inputs.prerelease_suffix }} - run: | - set -euo pipefail - - CURRENT_VERSION=$(node -p "require('./package.json').version") - RELEASE_COMMIT=$(git rev-parse HEAD) - - if [[ -z "${PRERELEASE_SUFFIX}" ]]; then - PRERELEASE_SUFFIX="${GITHUB_RUN_NUMBER}" - fi - - echo "release_type=${RELEASE_TYPE}" >> "$GITHUB_OUTPUT" - echo "branch=${TARGET_BRANCH}" >> "$GITHUB_OUTPUT" - echo "commit=${RELEASE_COMMIT}" >> "$GITHUB_OUTPUT" - - if [[ "$RELEASE_TYPE" == "stable" ]]; then - RELEASE_TAG="js-${CURRENT_VERSION}" - - if git ls-remote --exit-code --tags origin "refs/tags/${RELEASE_TAG}" >/dev/null 2>&1; then - echo "Tag ${RELEASE_TAG} already exists on origin" >&2 - exit 1 - fi - - echo "version=${CURRENT_VERSION}" >> "$GITHUB_OUTPUT" - echo "release_tag=${RELEASE_TAG}" >> "$GITHUB_OUTPUT" - else - VERSION="${CURRENT_VERSION}-rc.${PRERELEASE_SUFFIX}" - - echo "version=${VERSION}" >> "$GITHUB_OUTPUT" - echo "release_tag=" >> "$GITHUB_OUTPUT" - fi - - publish: - needs: prepare-release - runs-on: ubuntu-latest - timeout-minutes: 20 - permissions: - contents: write - id-token: write - env: - PACKAGE_NAME: autoevals - VERSION: ${{ needs.prepare-release.outputs.version }} - RELEASE_TAG: ${{ needs.prepare-release.outputs.release_tag }} - RELEASE_TYPE: ${{ needs.prepare-release.outputs.release_type }} - TARGET_BRANCH: ${{ needs.prepare-release.outputs.branch }} - RELEASE_COMMIT: ${{ needs.prepare-release.outputs.commit }} - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - fetch-depth: 0 - ref: ${{ needs.prepare-release.outputs.branch }} - - - name: Check version sync - run: python3 .github/scripts/check_version_sync.py - - - name: Set up Node.js - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 - with: - node-version-file: .tool-versions - registry-url: https://registry.npmjs.org - - - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 - with: - version: 10.33.0 - - - name: Check npm version availability - run: | - set -euo pipefail - - if npm view "${PACKAGE_NAME}@${VERSION}" version --registry=https://registry.npmjs.org >/dev/null 2>&1; then - echo "${PACKAGE_NAME}@${VERSION} already exists on npm" >&2 - exit 1 - fi - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Prepare prerelease package metadata - if: ${{ env.RELEASE_TYPE == 'prerelease' }} - run: | - set -euo pipefail - - node -e ' - const fs = require("fs"); - const pkg = JSON.parse(fs.readFileSync("package.json", "utf8")); - pkg.version = process.env.VERSION; - fs.writeFileSync("package.json", JSON.stringify(pkg, null, 2) + "\n"); - ' - - - name: Build package - run: pnpm run build - - - name: Publish stable release to npm - if: ${{ env.RELEASE_TYPE == 'stable' }} - env: - NODE_AUTH_TOKEN: "" - NPM_TOKEN: "" - run: npm publish --provenance --access public - - - name: Publish prerelease to npm - if: ${{ env.RELEASE_TYPE == 'prerelease' }} - env: - NODE_AUTH_TOKEN: "" - NPM_TOKEN: "" - run: npm publish --tag rc --provenance --access public - - - name: Create and push stable release tag - if: ${{ env.RELEASE_TYPE == 'stable' }} - run: | - set -euo pipefail - - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git tag "${RELEASE_TAG}" "${RELEASE_COMMIT}" - git push origin "${RELEASE_TAG}" - - - name: Create GitHub release - if: ${{ env.RELEASE_TYPE == 'stable' }} - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - env: - RELEASE_TAG: ${{ env.RELEASE_TAG }} - VERSION: ${{ env.VERSION }} - with: - script: | - await github.rest.repos.createRelease({ - owner: context.repo.owner, - repo: context.repo.repo, - tag_name: process.env.RELEASE_TAG, - name: `autoevals JavaScript v${process.env.VERSION}`, - draft: false, - prerelease: false, - generate_release_notes: true, - }); - - - name: Summarize release - run: | - set -euo pipefail - - { - echo "## npm publish complete" - echo - echo "- Package: \`${PACKAGE_NAME}\`" - echo "- Version: \`${VERSION}\`" - echo "- Release type: \`${RELEASE_TYPE}\`" - if [ "${RELEASE_TYPE}" = "prerelease" ]; then - echo "- npm tag: \`rc\`" - echo "- Install: \`npm install ${PACKAGE_NAME}@rc\`" - else - echo "- Git tag: \`${RELEASE_TAG}\`" - echo "- Install: \`npm install ${PACKAGE_NAME}\`" - fi - } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/publish-py.yaml b/.github/workflows/publish-py.yaml deleted file mode 100644 index 0cee9ab9..00000000 --- a/.github/workflows/publish-py.yaml +++ /dev/null @@ -1,233 +0,0 @@ -name: publish-py - -concurrency: - group: publish-py-${{ inputs.release_type }}-${{ inputs.branch }} - cancel-in-progress: false - -on: - workflow_dispatch: - inputs: - release_type: - description: Release type - required: true - default: stable - type: choice - options: - - stable - - prerelease - branch: - description: Branch to release from - required: true - default: main - type: string - prerelease_suffix: - description: Optional shared prerelease suffix - required: false - default: "" - type: string - -jobs: - prepare-release: - runs-on: ubuntu-latest - timeout-minutes: 10 - outputs: - version: ${{ steps.release_metadata.outputs.version }} - release_tag: ${{ steps.release_metadata.outputs.release_tag }} - branch: ${{ steps.release_metadata.outputs.branch }} - commit: ${{ steps.release_metadata.outputs.commit }} - release_type: ${{ steps.release_metadata.outputs.release_type }} - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - fetch-depth: 1 - ref: ${{ inputs.branch }} - - - name: Check version sync - run: python3 .github/scripts/check_version_sync.py - - - name: Set up Python - uses: actions/setup-python@3542bca2639a428e1796aaa6a2ffef0c0f575566 # v3.1.4 - with: - python-version: "3.12" - - - name: Determine release metadata - id: release_metadata - env: - RELEASE_TYPE: ${{ inputs.release_type }} - TARGET_BRANCH: ${{ inputs.branch }} - PRERELEASE_SUFFIX: ${{ inputs.prerelease_suffix }} - run: | - set -euo pipefail - - CURRENT_VERSION=$(python -c 'from pathlib import Path; ns = {}; exec(Path("py/autoevals/version.py").read_text(encoding="utf-8"), ns); print(ns["VERSION"])') - RELEASE_COMMIT=$(git rev-parse HEAD) - - if [[ -z "${PRERELEASE_SUFFIX}" ]]; then - PRERELEASE_SUFFIX="${GITHUB_RUN_NUMBER}" - fi - - echo "release_type=${RELEASE_TYPE}" >> "$GITHUB_OUTPUT" - echo "branch=${TARGET_BRANCH}" >> "$GITHUB_OUTPUT" - echo "commit=${RELEASE_COMMIT}" >> "$GITHUB_OUTPUT" - - if [[ "$RELEASE_TYPE" == "stable" ]]; then - RELEASE_TAG="py-${CURRENT_VERSION}" - - if git ls-remote --exit-code --tags origin "refs/tags/${RELEASE_TAG}" >/dev/null 2>&1; then - echo "Tag ${RELEASE_TAG} already exists on origin" >&2 - exit 1 - fi - - echo "version=${CURRENT_VERSION}" >> "$GITHUB_OUTPUT" - echo "release_tag=${RELEASE_TAG}" >> "$GITHUB_OUTPUT" - else - VERSION="${CURRENT_VERSION}rc${PRERELEASE_SUFFIX}" - - echo "version=${VERSION}" >> "$GITHUB_OUTPUT" - echo "release_tag=" >> "$GITHUB_OUTPUT" - fi - - publish: - needs: prepare-release - runs-on: ubuntu-latest - timeout-minutes: 20 - permissions: - contents: write - id-token: write # Required for PyPI trusted publishing - env: - PACKAGE_NAME: autoevals - VERSION: ${{ needs.prepare-release.outputs.version }} - RELEASE_TAG: ${{ needs.prepare-release.outputs.release_tag }} - RELEASE_TYPE: ${{ needs.prepare-release.outputs.release_type }} - TARGET_BRANCH: ${{ needs.prepare-release.outputs.branch }} - RELEASE_COMMIT: ${{ needs.prepare-release.outputs.commit }} - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - fetch-depth: 0 - ref: ${{ needs.prepare-release.outputs.branch }} - - - name: Check version sync - run: python3 .github/scripts/check_version_sync.py - - - name: Set up Python - uses: actions/setup-python@3542bca2639a428e1796aaa6a2ffef0c0f575566 # v3.1.4 - with: - python-version: "3.12" - - - name: Check PyPI version availability - run: | - set -euo pipefail - - python - <<'PY' - import os - import sys - import urllib.error - import urllib.request - - package = os.environ["PACKAGE_NAME"] - version = os.environ["VERSION"] - url = f"https://pypi.org/pypi/{package}/{version}/json" - - try: - urllib.request.urlopen(url) - except urllib.error.HTTPError as exc: - if exc.code == 404: - raise SystemExit(0) - raise - except urllib.error.URLError as exc: - print(f"Failed to query PyPI: {exc}", file=sys.stderr) - raise - else: - print(f"{package}=={version} already exists on PyPI", file=sys.stderr) - raise SystemExit(1) - PY - - - name: Set up uv - uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 - with: - enable-cache: true - - - name: Install build dependencies - run: uv sync --extra dev - - - name: Prepare prerelease package metadata - if: ${{ env.RELEASE_TYPE == 'prerelease' }} - run: | - set -euo pipefail - - python - <<'PY' - import os - import re - from pathlib import Path - - path = Path("py/autoevals/version.py") - text = path.read_text(encoding="utf-8") - new_text, count = re.subn( - r'^VERSION\s*=\s*["\'][^"\']+["\']\s*$', - f'VERSION = "{os.environ["VERSION"]}"', - text, - count=1, - flags=re.MULTILINE, - ) - if count != 1: - raise SystemExit("Could not update py/autoevals/version.py for prerelease publish") - path.write_text(new_text + ("" if new_text.endswith("\n") else "\n"), encoding="utf-8") - PY - - - name: Build package - run: uv run --extra dev python -m build - - - name: Verify package metadata - run: uv run --extra dev python -m twine check dist/* - - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 - with: - packages-dir: dist/ - - - name: Create and push stable release tag - if: ${{ env.RELEASE_TYPE == 'stable' }} - run: | - set -euo pipefail - - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git tag "${RELEASE_TAG}" "${RELEASE_COMMIT}" - git push origin "${RELEASE_TAG}" - - - name: Create GitHub release - if: ${{ env.RELEASE_TYPE == 'stable' }} - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - env: - RELEASE_TAG: ${{ env.RELEASE_TAG }} - VERSION: ${{ env.VERSION }} - with: - script: | - await github.rest.repos.createRelease({ - owner: context.repo.owner, - repo: context.repo.repo, - tag_name: process.env.RELEASE_TAG, - name: `autoevals Python v${process.env.VERSION}`, - draft: false, - prerelease: false, - generate_release_notes: true, - }); - - - name: Summarize release - run: | - set -euo pipefail - - { - echo "## PyPI publish complete" - echo - echo "- Package: \`${PACKAGE_NAME}\`" - echo "- Version: \`${VERSION}\`" - echo "- Release type: \`${RELEASE_TYPE}\`" - if [ "${RELEASE_TYPE}" = "prerelease" ]; then - echo "- Install: \`pip install --pre ${PACKAGE_NAME}\`" - else - echo "- Git tag: \`${RELEASE_TAG}\`" - echo "- Install: \`pip install ${PACKAGE_NAME}\`" - fi - } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 55393bd3..ffdf6ce2 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -1,66 +1,349 @@ +# Releases the autoevals npm + PyPI packages TOGETHER, at one shared version, behind ONE +# approval gate — via braintrustdata/sdk-actions (custom / build-ownership shape). +# +# Shape (fan-out → fan-in → fan-out), one lane per package: +# configure-{js,py} → validate-{js,py} → pack-{js,py} (unprivileged build+attest) +# └──────────────── request-approval (single gate: builds one approval covering +# both packages, then parks at the `publish` environment) +# request-approval → ship-package-{js,py} (gated: verify attestation → publish → GitHub release) +# +# configure is the fact-finding action for each package: it derives version/tag/channel, +# checks the registry, fetches release notes, and emits a `package` JSON object. validate +# only judges those facts; request-approval consumes the package objects directly. +# +# This is hand-authored, but deliberately modeled as the output a future sdk-actions "monorepo" +# template should generate: each lane is an independent, uniformly-parameterized column; the only +# shared, once-per-release job is request-approval, which composes a `packages` array (one entry per +# lane, from configure outputs) so reviewers approve BOTH packages in one gate. A failure in +# either lane skips request-approval → neither ships (the packages always release together). +# +# ── SETUP (repo Settings) ───────────────────────────────────────────────────────────────────── +# Environments: `publish` (required reviewers) and `publish-dry-run`. +# Trusted publishers (OIDC, no tokens) — BOTH must set environment=`publish`, which is what keeps +# the unprivileged pack jobs (id-token for attestation only) from being able to publish: +# npm `autoevals` → repo braintrustdata/autoevals, workflow publish.yaml, environment publish +# PyPI `autoevals` → repo braintrustdata/autoevals, workflow publish.yaml, environment publish +# Secrets/vars (optional Slack): SLACK_BOT_TOKEN, SLACK_SDK_RELEASE_CHANNEL. +# +# Version-sync (package.json == py/autoevals/version.py) is enforced by the version-sync.yaml CI +# workflow on PRs, so it is intentionally NOT re-checked here. Prereleases publish the COMMITTED +# version (release_type only suppresses the GitHub release / sets the npm rc channel) — bump to an +# rc version in a PR if you want a prerelease. +# +# sdk-actions pinned @2834359 (branch release_request_approval_multi_package, off #63). RE-PIN to the +# merged main SHA before merging this PR. name: publish concurrency: - group: publish-${{ inputs.release_type }}-${{ inputs.branch }} + group: publish-${{ inputs.release_type }}-${{ inputs.sha }} cancel-in-progress: false on: workflow_dispatch: inputs: + _instructions: + description: "⚠️ Before starting: merge a version-bump PR (package.json + py/autoevals/version.py, matching). The release publishes the version at the SHA — it can't be overridden here." + type: string + default: "I have merged a version bump PR" + required: false release_type: - description: Release type - required: true - default: stable + description: "Release type (stable → GitHub releases; prerelease → none, npm rc channel)" type: choice - options: - - stable - - prerelease - branch: - description: Branch to publish from + default: stable + options: [stable, prerelease] + sha: + description: "Commit SHA (of the version bump) to release — same commit for js + py" required: true - default: main type: string + prev_release: + description: "Release-notes anchor: a tag or SHA (empty → each lane's previous tag)" + type: string + required: false + dry_run: + description: "Dry run: build + pack + attest, but do not publish or create releases" + type: boolean + default: false jobs: - dispatch-package-publishes: - runs-on: ubuntu-latest + # ─────────────────────────── JS lane ─────────────────────────── + configure-js: + runs-on: ubuntu-24.04 timeout-minutes: 5 permissions: - actions: write + contents: write # releases/generate-notes API + outputs: + version: ${{ steps.configure.outputs.version }} + release_tag: ${{ steps.configure.outputs.release_tag }} + prev_release: ${{ steps.configure.outputs.prev_release }} + branch: ${{ steps.configure.outputs.branch }} + on_release_branch: ${{ steps.configure.outputs.on_release_branch }} + commit_message: ${{ steps.configure.outputs.commit_message }} + channel: ${{ steps.configure.outputs.channel }} + github_release: ${{ steps.configure.outputs.github_release }} + already_published: ${{ steps.configure.outputs.already_published }} + notes: ${{ steps.configure.outputs.notes }} + package: ${{ steps.configure.outputs.package }} + steps: + - name: Configure release + id: configure + uses: braintrustdata/sdk-actions/actions/release/lang/js/configure@2834359ac1151454c3479076552ce72e7737a808 + with: + sha: ${{ inputs.sha }} + working_directory: . + tag_format: "js-{version}" + release_type: ${{ inputs.release_type }} + npm_package_name: autoevals + package_label: "autoevals (js)" + emoji: ":javascript:" + prev_release: ${{ inputs.prev_release }} + + validate-js: + needs: [configure-js] + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + steps: + - name: Validate release + uses: braintrustdata/sdk-actions/actions/release/lang/js/validate@2834359ac1151454c3479076552ce72e7737a808 + with: + sha: ${{ inputs.sha }} + dry_run: ${{ inputs.dry_run }} + working_directory: . + node_version: .tool-versions + release_tag: ${{ needs.configure-js.outputs.release_tag }} + channel: ${{ needs.configure-js.outputs.channel }} + allowed_channels: "latest,rc" + on_release_branch: ${{ needs.configure-js.outputs.on_release_branch }} + notes: ${{ needs.configure-js.outputs.notes }} + already_published: ${{ needs.configure-js.outputs.already_published }} + build: "false" # pack-js owns the build + sbom: "false" # pack-js generates + attests the SBOM + # Multi-package heal re-run: don't hard-abort when one package/tag is already out; + # ship-package skips the lane that's already published and releases the one that failed. + check_version_unpublished: "false" + check_tag_unused: "false" + + pack-js: + needs: [configure-js, validate-js] + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + id-token: write + attestations: write + steps: + - name: Pack + attest + id: pack + uses: braintrustdata/sdk-actions/actions/release/lang/js/pack-pnpm@2834359ac1151454c3479076552ce72e7737a808 + with: + sha: ${{ inputs.sha }} + working_directory: . + node_version: .tool-versions + # autoevals installs + builds on its pinned pnpm 10.33 (to honor its supply-chain config); + # the packer resolves a pnpm >= 11.8 on its own for `pnpm pack` + the SBOM. No override here. + dry_run: ${{ inputs.dry_run }} + + - name: Stage artifact + run: | + mkdir -p artifact + cp "${{ steps.pack.outputs.tarball }}" artifact/ + cp ./sbom.json artifact/ 2>/dev/null || true + + - name: Upload artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-package-js + path: artifact/ + retention-days: 1 + if-no-files-found: error + + # ─────────────────────────── PY lane ─────────────────────────── + configure-py: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: write # releases/generate-notes API + outputs: + version: ${{ steps.configure.outputs.version }} + release_tag: ${{ steps.configure.outputs.release_tag }} + prev_release: ${{ steps.configure.outputs.prev_release }} + branch: ${{ steps.configure.outputs.branch }} + on_release_branch: ${{ steps.configure.outputs.on_release_branch }} + commit_message: ${{ steps.configure.outputs.commit_message }} + github_release: ${{ steps.configure.outputs.github_release }} + already_published: ${{ steps.configure.outputs.already_published }} + notes: ${{ steps.configure.outputs.notes }} + package: ${{ steps.configure.outputs.package }} + steps: + - name: Configure release + id: configure + uses: braintrustdata/sdk-actions/actions/release/lang/py/configure@2834359ac1151454c3479076552ce72e7737a808 + with: + version_file: py/autoevals/version.py + sha: ${{ inputs.sha }} + working_directory: . + tag_format: "py-{version}" + release_type: ${{ inputs.release_type }} + pypi_package_name: autoevals + package_label: "autoevals (py)" + emoji: ":python:" + prev_release: ${{ inputs.prev_release }} + + validate-py: + needs: [configure-py] + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read steps: - - name: Dispatch JS and Python publish workflows - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - env: - RELEASE_TYPE: ${{ inputs.release_type }} - TARGET_BRANCH: ${{ inputs.branch }} - PRERELEASE_SUFFIX: ${{ github.run_number }} - WORKFLOW_REF: main + - name: Validate release + uses: braintrustdata/sdk-actions/actions/release/lang/py/validate@2834359ac1151454c3479076552ce72e7737a808 with: - script: | - const workflows = ["publish-js.yaml", "publish-py.yaml"]; - for (const workflow_id of workflows) { - await github.rest.actions.createWorkflowDispatch({ - owner: context.repo.owner, - repo: context.repo.repo, - workflow_id, - ref: process.env.WORKFLOW_REF, - inputs: { - release_type: process.env.RELEASE_TYPE, - branch: process.env.TARGET_BRANCH, - prerelease_suffix: process.env.PRERELEASE_SUFFIX, - }, - }); - } - - - name: Summarize dispatch + sha: ${{ inputs.sha }} + dry_run: ${{ inputs.dry_run }} + working_directory: . + python_version: .tool-versions + release_tag: ${{ needs.configure-py.outputs.release_tag }} + on_release_branch: ${{ needs.configure-py.outputs.on_release_branch }} + notes: ${{ needs.configure-py.outputs.notes }} + already_published: ${{ needs.configure-py.outputs.already_published }} + build: "false" # pack-py owns the build + sbom: "false" # pack-py generates + attests the SBOM + # Multi-package heal re-run: don't hard-abort when one package/tag is already out; + # ship-package skips the lane that's already published and releases the one that failed. + check_version_unpublished: "false" + check_tag_unused: "false" + + pack-py: + needs: [configure-py, validate-py] + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + id-token: write + attestations: write + steps: + - name: Pack + attest + uses: braintrustdata/sdk-actions/actions/release/lang/py/pack@2834359ac1151454c3479076552ce72e7737a808 + with: + sha: ${{ inputs.sha }} + working_directory: . + python_version: .tool-versions + dry_run: ${{ inputs.dry_run }} + + - name: Stage artifact run: | - { - echo "## Package publishes queued" - echo - echo "- Workflows: \`publish-js.yaml\`, \`publish-py.yaml\`" - echo "- Release type: \`${{ inputs.release_type }}\`" - echo "- Branch: \`${{ inputs.branch }}\`" - if [ "${{ inputs.release_type }}" = "prerelease" ]; then - echo "- Shared prerelease suffix: \`${{ github.run_number }}\`" - fi - } >> "$GITHUB_STEP_SUMMARY" + mkdir -p artifact/dist + cp ./dist/*.whl ./dist/*.tar.gz artifact/dist/ + cp ./sbom.json artifact/ 2>/dev/null || true + + - name: Upload artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-package-py + path: artifact/ + retention-days: 1 + if-no-files-found: error + + # ─────────────── fan-in: ONE approval covering both packages ─────────────── + request-approval: + needs: + [configure-js, validate-js, pack-js, configure-py, validate-py, pack-py] + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: {} + steps: + - name: Request release approval + uses: braintrustdata/sdk-actions/actions/release/request-approval@2834359ac1151454c3479076552ce72e7737a808 + with: + packages: "[${{ needs.configure-js.outputs.package }}, ${{ needs.configure-py.outputs.package }}]" + title: autoevals + sha: ${{ inputs.sha }} + branch: ${{ needs.configure-js.outputs.branch }} + on_release_branch: ${{ needs.configure-js.outputs.on_release_branch }} + commit_message: ${{ needs.configure-js.outputs.commit_message }} + dry_run: ${{ inputs.dry_run }} + slack_token: ${{ secrets.SLACK_BOT_TOKEN }} + slack_channel: ${{ vars.SLACK_SDK_RELEASE_CHANNEL }} + + # ─────────────── fan-out: gated per-lane publish (one approval releases both) ─────────────── + ship-package-js: + needs: [configure-js, pack-js, request-approval] + runs-on: ubuntu-24.04 + timeout-minutes: 15 + environment: ${{ inputs.dry_run && 'publish-dry-run' || 'publish' }} + permissions: + contents: write + id-token: write # OIDC trusted publishing + provenance + attestations: read # gh attestation verify + steps: + - name: Download artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: release-package-js + path: dl + + - name: Resolve tarball + id: art + run: echo "tarball=$(ls dl/*.tgz)" >> "$GITHUB_OUTPUT" + + - name: Ship package + uses: braintrustdata/sdk-actions/actions/release/lang/js/ship-package@2834359ac1151454c3479076552ce72e7737a808 + with: + tarball: ${{ steps.art.outputs.tarball }} + working_directory: dl + sha: ${{ inputs.sha }} + dry_run: ${{ inputs.dry_run }} + release_tag: ${{ needs.configure-js.outputs.release_tag }} + already_published: ${{ needs.configure-js.outputs.already_published }} + github_release: ${{ needs.configure-js.outputs.github_release }} + version: ${{ needs.configure-js.outputs.version }} + access: "public" + provenance: "true" + channel: ${{ needs.configure-js.outputs.channel }} + package_name: autoevals + label: "autoevals (js)" + notes: ${{ needs.configure-js.outputs.notes }} + prev_release: ${{ needs.configure-js.outputs.prev_release }} + branch: ${{ needs.configure-js.outputs.branch }} + on_release_branch: ${{ needs.configure-js.outputs.on_release_branch }} + slack_token: ${{ secrets.SLACK_BOT_TOKEN }} + slack_channel: ${{ vars.SLACK_SDK_RELEASE_CHANNEL }} + emoji: ":javascript:" + + ship-package-py: + needs: [configure-py, pack-py, request-approval] + runs-on: ubuntu-24.04 + timeout-minutes: 15 + environment: ${{ inputs.dry_run && 'publish-dry-run' || 'publish' }} + permissions: + contents: write + id-token: write # OIDC trusted publishing + PEP 740 + attestations: read # gh attestation verify + steps: + - name: Download artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: release-package-py + path: dl + + - name: Ship package + uses: braintrustdata/sdk-actions/actions/release/lang/py/ship-package@2834359ac1151454c3479076552ce72e7737a808 + with: + working_directory: dl # holds dist/ (whl + sdist) + sbom.json + sha: ${{ inputs.sha }} + dry_run: ${{ inputs.dry_run }} + release_tag: ${{ needs.configure-py.outputs.release_tag }} + already_published: ${{ needs.configure-py.outputs.already_published }} + github_release: ${{ needs.configure-py.outputs.github_release }} + version: ${{ needs.configure-py.outputs.version }} + package_name: autoevals + label: "autoevals (py)" + notes: ${{ needs.configure-py.outputs.notes }} + prev_release: ${{ needs.configure-py.outputs.prev_release }} + branch: ${{ needs.configure-py.outputs.branch }} + on_release_branch: ${{ needs.configure-py.outputs.on_release_branch }} + slack_token: ${{ secrets.SLACK_BOT_TOKEN }} + slack_channel: ${{ vars.SLACK_SDK_RELEASE_CHANNEL }} + emoji: ":python:" diff --git a/docs/PUBLISHING.md b/docs/PUBLISHING.md index ab94fb00..731f3cc9 100644 --- a/docs/PUBLISHING.md +++ b/docs/PUBLISHING.md @@ -1,211 +1,51 @@ # Publishing -This repository contains both JavaScript and Python packages, both published as `autoevals`: +Publishes the npm and PyPI `autoevals` packages **together, at one shared version, behind one +approval gate** — via a single workflow, `.github/workflows/publish.yaml`, using the shared release +actions from [`braintrustdata/sdk-actions`](https://github.com/braintrustdata/sdk-actions) (pinned by SHA). -- npm package: `autoevals` -- PyPI package: `autoevals` +## Versioning -Publishing is handled via GitHub Actions trusted publishing. +`package.json` and `py/autoevals/version.py` must hold the same version; `version-sync.yaml` (CI) +enforces it on PRs. The release publishes the **committed** version — bump both and merge first. -## Workflows +## Releasing -Publishing workflows: +In GitHub Actions, run **`publish`** with: -- `.github/workflows/publish.yaml` — manual dispatcher that triggers both package publish workflows -- `.github/workflows/publish-js.yaml` — npm publish workflow -- `.github/workflows/publish-py.yaml` — PyPI publish workflow -- `.github/workflows/version-sync.yaml` — CI check that JS/Python versions stay in sync +- `release_type` — `stable` (default) or `prerelease` +- `sha` — full 40-char commit SHA to release (same commit for js + py) +- `prev_release` (optional) — release-notes anchor; empty → each lane's previous tag +- `dry_run` (default `false`) — build + pack + attest, no publish or release -## Versioning policy +One approval releases both packages; if either lane fails before the gate, neither ships. -JavaScript and Python package versions must always match. +- **stable** → npm `autoevals@` (provenance) + PyPI `autoevals==` (attestation); tags `js-` + and `py-`; two GitHub Releases. +- **prerelease** → publishes the committed version under the npm `rc` dist-tag; no GitHub release. -The canonical version files are: +### Steps -- `package.json` -- `py/autoevals/version.py` +1. Bump `package.json` + `py/autoevals/version.py` together and merge to `main`. +2. Run `publish` with `release_type=stable`, `sha=`. +3. Approve the `publish` environment when the run parks at the gate. -CI enforces this with: +## Requirements & notes -- `.github/workflows/version-sync.yaml` -- `.github/scripts/check_version_sync.py` +- **Environments** `publish` (required reviewers) and `publish-dry-run` must exist (repo Settings). +- **Trusted publishers** for npm `autoevals` and PyPI `autoevals` must be configured with **environment + `publish`** (OIDC — no tokens). The environment claim is what confines publishing to the gated ship + jobs; without it, publishing fails. A `dry_run` never publishes, so the first real publish is the + first true test of this config. +- **pnpm**: install + build run on the repo's pinned `pnpm@10.33.0` (honoring the supply-chain + config in `pnpm-workspace.yaml`). The packer automatically uses a pnpm ≥ 11.8 for `pnpm pack` and + the SBOM — those two commands require it — so no release-specific pnpm setup is needed here. +- **Slack** notifications are optional: variable `SLACK_SDK_RELEASE_CHANNEL`, secret `SLACK_BOT_TOKEN`. -If these versions do not match, CI fails and publish workflows fail early. - -## Recommended publish flow - -Use the top-level `publish` workflow for normal releases. - -In GitHub Actions, manually run: - -- `publish` - -Inputs: - -- `release_type=stable` or `prerelease` -- `branch=main` (or another branch to publish from) - -This workflow dispatches both: - -- `publish-js.yaml` -- `publish-py.yaml` - -For prereleases, the dispatcher passes a shared prerelease suffix to both workflows so the releases stay aligned: - -- npm: `-rc.` -- PyPI: `rc` - -Example for base version `0.2.0` and suffix `123`: - -- npm: `0.2.0-rc.123` -- PyPI: `0.2.0rc123` - -## JavaScript npm publishing - -The JavaScript publish workflow lives at: - -- `.github/workflows/publish-js.yaml` - -It supports two release types: - -- `stable`: publishes the exact version in `package.json` -- `prerelease`: publishes `-rc.` with the `rc` dist-tag - -For stable releases, the workflow also: - -- creates and pushes a git tag named `js-` -- creates a GitHub Release named `autoevals JavaScript v` - -### npm trusted publishing setup - -Configure trusted publishing for the `autoevals` package in npm with these values: - -- Package: `autoevals` -- Provider: `GitHub Actions` -- Repository owner: `braintrustdata` -- Repository name: `autoevals` -- Workflow file: `.github/workflows/publish-js.yaml` - -Notes: - -- The workflow uses GitHub OIDC, so no `NPM_TOKEN` is required. -- The workflow publishes with provenance enabled via `npm publish --provenance`. - -## Python PyPI publishing - -The Python publish workflow lives at: - -- `.github/workflows/publish-py.yaml` - -It supports two release types: - -- `stable`: publishes the exact version in `py/autoevals/version.py` -- `prerelease`: publishes a PEP 440 prerelease version `rc` - -For stable releases, the workflow also: - -- creates and pushes a git tag named `py-` -- creates a GitHub Release named `autoevals Python v` - -### PyPI trusted publishing setup - -Configure trusted publishing for the `autoevals` project in PyPI with these values: - -- Project name: `autoevals` -- Owner: `braintrustdata` -- Repository name: `autoevals` -- Workflow file: `.github/workflows/publish-py.yaml` - -Notes: - -- The workflow uses GitHub OIDC, so no PyPI API token is required. -- The workflow publishes via `pypa/gh-action-pypi-publish`. -- The workflow must have `id-token: write` permission for trusted publishing. - -## How to publish a stable release - -1. Bump both versions together: - - `package.json` - - `py/autoevals/version.py` -2. Merge the change to `main`. -3. In GitHub Actions, run the `publish` workflow. -4. Choose: - - `release_type=stable` - - `branch=main` - -Expected outcome: - -- npm package `autoevals@` is published -- PyPI package `autoevals==` is published -- git tag `js-` is created and pushed -- git tag `py-` is created and pushed -- GitHub Release `autoevals JavaScript v` is created -- GitHub Release `autoevals Python v` is created - -## How to publish a prerelease - -1. Make sure both version files contain the same base version: - - `package.json` - - `py/autoevals/version.py` -2. In GitHub Actions, run the `publish` workflow. -3. Choose: - - `release_type=prerelease` - - `branch=main` - -Expected outcome: - -- npm package `autoevals@-rc.` is published -- npm dist-tag `rc` is updated -- PyPI package `autoevals==rc` is published -- no stable git tags are created -- no GitHub Releases are created - -## Publishing package-specific workflows directly - -If needed, you can manually trigger either workflow directly: - -- `publish-js` -- `publish-py` - -Both accept: - -- `release_type` -- `branch` -- `prerelease_suffix` (optional) - -Normally you should prefer the top-level `publish` workflow so JS and Python prereleases use the same suffix. - -## Safeguards in the workflows - -The workflows fail early if: - -- `package.json` and `py/autoevals/version.py` do not match -- the stable JS tag `js-` already exists on `origin` -- the stable Python tag `py-` already exists on `origin` -- the npm version being published already exists -- the PyPI version being published already exists - -## Local validation - -Useful commands before triggering a release: +## Local checks ```bash python3 .github/scripts/check_version_sync.py -pnpm install --frozen-lockfile -pnpm run build -npm publish --dry-run --access public -uv sync --extra dev -uv run python -m build -uv run python -m twine check dist/* +pnpm install --frozen-lockfile && pnpm run build +uv build && uvx twine check dist/* ``` - -## Historical releases and source mapping - -Older npm releases may not be traceable back to an exact git commit from npm alone because they were published before trusted publishing and provenance attestations were enabled. In particular: - -- npm metadata for older releases may not include `gitHead` -- those releases do not have OIDC/provenance attestations tying the package to a workflow run and commit - -For those historical versions, the best commit mapping may need to be inferred from repository history, publish timestamps, and version bumps. New npm releases published through `.github/workflows/publish-js.yaml` are easier to trace because they use trusted publishing with provenance. - -Python releases published through `.github/workflows/publish-py.yaml` are similarly expected to be easier to trace because they use PyPI trusted publishing via GitHub Actions OIDC.