Sync dry-run from dist-$STACK-${env.src_path_suffix} to dist-$STACK-${env.dst_path_suffix} #87
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Platform packages sync from -cnb/ to -cnb-stable/ | |
| run-name: Sync${{ inputs.dry-run == true && ' dry-run' || '' }} from dist-$STACK-${env.src_path_suffix} to dist-$STACK-${env.dst_path_suffix} | |
| env: | |
| stacks_list_for_shell_expansion: "{heroku-20,heroku-22,heroku-24-amd64,heroku-24-arm64}" | |
| src_path_suffix: "-cnb/" | |
| dst_path_suffix: "-cnb-stable/" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| stack-heroku-20: | |
| description: 'Sync heroku-20 packages' | |
| type: boolean | |
| default: true | |
| required: false | |
| stack-heroku-22: | |
| description: 'Sync heroku-22 packages' | |
| type: boolean | |
| default: true | |
| required: false | |
| stack-heroku-24-amd64: | |
| description: 'Sync heroku-24 (amd64) packages' | |
| type: boolean | |
| default: true | |
| required: false | |
| stack-heroku-24-arm64: | |
| description: 'Sync heroku-24 (arm64) packages' | |
| type: boolean | |
| default: true | |
| required: false | |
| dry-run: | |
| description: 'Only list package changes, without syncing' | |
| type: boolean | |
| default: false | |
| required: false | |
| update-devcenter: | |
| description: 'Update "PHP Support" Dev Center article after (non dry-run) sync' | |
| type: boolean | |
| default: true | |
| required: false | |
| permissions: | |
| contents: read | |
| jobs: | |
| stack-list: | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| stacks: ${{ steps.list-stacks.outputs.matrix }} | |
| steps: | |
| - id: list-stacks | |
| name: Generate list of stacks to sync based on input checkboxes | |
| run: | | |
| echo '## Stacks to sync' >> "$GITHUB_STEP_SUMMARY" | |
| set -o pipefail | |
| stacks=(${{ inputs.stack-heroku-20 == true && 'heroku-20' || ''}} ${{ inputs.stack-heroku-22 == true && 'heroku-22' || ''}} ${{ inputs.stack-heroku-24-amd64 == true && 'heroku-24-amd64' || ''}} ${{ inputs.stack-heroku-24-arm64 == true && 'heroku-24-arm64' || ''}}) | |
| printf -- "- %s\n" "${stacks[@]}" >> "$GITHUB_STEP_SUMMARY" | |
| echo -n "matrix=" >> "$GITHUB_OUTPUT" | |
| printf "%s\n" "${stacks[@]}" | jq -jcRn '[inputs|select(length>0)]' >> "$GITHUB_OUTPUT" | |
| docker-build: | |
| needs: stack-list | |
| if: ${{ needs.stack-list.outputs.stacks != '[]' && needs.stack-list.outputs.stacks != '' }} | |
| runs-on: ${{ endsWith(matrix.stack, '-arm64') && 'pub-hk-ubuntu-22.04-arm-small' || 'ubuntu-22.04' }} | |
| strategy: | |
| matrix: | |
| stack: ${{ fromJSON(needs.stack-list.outputs.stacks) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Cache Docker build | |
| id: cache-docker | |
| uses: actions/cache@v4 | |
| with: | |
| key: docker-cache-heroku-php-build-${{matrix.stack}}.${{github.sha}} | |
| path: /tmp/docker-cache.tar.gz | |
| - name: Build Docker image | |
| if: steps.cache-docker.outputs.cache-hit != 'true' | |
| # our "input" stack might contain a "-amd64" or "-arm64" suffix, which we strip off for the Dockerfile name | |
| run: | | |
| shopt -s extglob | |
| stackname_with_architecture=${{matrix.stack}} | |
| docker build --tag heroku-php-build-${stackname_with_architecture}:${{github.sha}} --file support/build/_docker/${stackname_with_architecture%-?(amd|arm)64}.Dockerfile . | |
| - name: Save built Docker image | |
| if: steps.cache-docker.outputs.cache-hit != 'true' | |
| run: docker save heroku-php-build-${{matrix.stack}}:${{github.sha}} | gzip -1 > /tmp/docker-cache.tar.gz | |
| sync: | |
| needs: [stack-list, docker-build] | |
| strategy: | |
| matrix: | |
| stack: ${{ fromJSON(needs.stack-list.outputs.stacks) }} | |
| runs-on: ${{ endsWith(matrix.stack, '-arm64') && 'pub-hk-ubuntu-22.04-arm-small' || 'ubuntu-22.04' }} | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Restore cached Docker build | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: docker-cache-heroku-php-build-${{matrix.stack}}.${{github.sha}} | |
| path: /tmp/docker-cache.tar.gz | |
| - name: Load cached Docker image | |
| run: docker load -i /tmp/docker-cache.tar.gz | |
| - name: ${{ inputs.dry-run == true && 'Dry-run sync of' || 'Sync' }} changed packages to production bucket | |
| run: | | |
| # we want to fail if 'docker run' fails; without this, 'tee' would "eat" the failure status | |
| set -o pipefail | |
| # yes gets "n" to print for dry-runs so the sync aborts | |
| # errors are redirected to /dev/null, and we || true, to suppress SIGPIPE errors from 'docker run' exiting eventually | |
| # we need -i for Docker to accept input on stdin, but must not use -t for the pipeline to work | |
| (yes "${{ inputs.dry-run == true && 'n' || 'y' }}" 2>/dev/null || true) | docker run --rm -i --env-file=support/build/_docker/env.default heroku-php-build-${{matrix.stack}}:${{github.sha}} sync.sh lang-php dist-${{matrix.stack}}${env.dst_path_suffix} 2>&1 | tee sync-${{matrix.stack}}.log | |
| - name: Upload sync log as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: synclog-${{matrix.stack}} | |
| path: sync-${{matrix.stack}}.log | |
| - name: Output dry-run summary | |
| if: ${{ inputs.dry-run == true }} | |
| run: | | |
| echo '## Package changes available for syncing to ${{matrix.stack}} production bucket' >> "$GITHUB_STEP_SUMMARY" | |
| echo '**This is output from a dry-run**, no changes have been synced to production:' >> "$GITHUB_STEP_SUMMARY" | |
| echo '```' >> "$GITHUB_STEP_SUMMARY" | |
| sed -n '/^The following packages will/,/POTENTIALLY DESTRUCTIVE ACTION/{/POTENTIALLY DESTRUCTIVE ACTION/!p}' sync-${{matrix.stack}}.log >> "$GITHUB_STEP_SUMMARY" | |
| echo '```' >> "$GITHUB_STEP_SUMMARY" | |
| - name: Output sync summary | |
| if: ${{ inputs.dry-run == false }} | |
| run: | | |
| echo '## Package changes synced to ${{matrix.stack}} production bucket' >> "$GITHUB_STEP_SUMMARY" | |
| echo '```' >> "$GITHUB_STEP_SUMMARY" | |
| cat sync-${{matrix.stack}}.log >> "$GITHUB_STEP_SUMMARY" | |
| echo '```' >> "$GITHUB_STEP_SUMMARY" | |
| devcenter-generate: | |
| needs: sync | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| diff_result: ${{ steps.diff.outputs.diff_result }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install dos2unix | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install dos2unix | |
| - name: Install PHP and Composer | |
| uses: shivammathur/setup-php@v2 | |
| with: | |
| php-version: "8.2" | |
| tools: "composer:2.7" | |
| - name: Install Dev Center generator dependencies | |
| run: | | |
| composer install -d support/devcenter/ | |
| - name: Generate Dev Center article sections | |
| run: | | |
| set -o pipefail | |
| urls=( https://lang-php.s3.amazonaws.com/dist-${{ env.stacks_list_for_shell_expansion }}${{ inputs.dry-run == true && env.src_path_suffix || env.dst_path_suffix }}/packages.json ) | |
| # generate.php can generate individual sections, but doing it in one go a) is faster and b) means this code does not need to know what those sections are | |
| # Instead we split the generated contents into individual files, with the known delimiter as the split pattern. | |
| # tee stderr to a file | |
| support/devcenter/generate.php "${urls[@]}" 2> >(tee warnings.txt >&2) | csplit -s -z -f 'section-' -b '%02d.md' - '/^<!-- BEGIN [A-Z_][A-Z0-9_-]\+ -->$/' '{*}' | |
| # if there were warnings or notices on stderr, output them for GitHub highlighting | |
| # the \L, \E and \u extensions are case modifiers | |
| # this turns "(WARNING|NOTICE): foobar" into "::(warning|notice) title=(Warning|Notice) emitted by generator.php::foobar" | |
| sed -E -e 's/^(WARNING|NOTICE): /::\L\1\E title=\L\u\1\E emitted by generator.php::/g' warnings.txt | |
| # sanity check number of generated splits (e.g. in case the split ever changes) | |
| shopt -s nullglob | |
| splits=( section-*.md ) | |
| if (( ${#splits[@]} < 2 )); then | |
| echo 'error::Expected more than one section from generator.' | |
| exit 1 | |
| fi | |
| - name: Download current Dev Center article markdown | |
| run: | | |
| set -o pipefail | |
| # jq -j, not -r, otherwise we get a stray trailing newline | |
| curl -H "Accept: application/json" https://devcenter.heroku.com/api/v1/articles/php-support | jq -j '.content' > php-support.md | |
| # Because the articles are edited in a web interface, they likely use CRLF line endings. | |
| # We will be patching using the LF line ending section files generated in an earlier step. | |
| # For this reason, we may have to convert to LF, so we check if the file would be converted by dos2unix using the --info option. | |
| # The "c" info flag prints only file names that would trigger conversion; we first remember this output for the next step via tee. | |
| # The "0" flag triggers zero-byte output for happy consumption by xargs | |
| # Then, we finally run the conversion (if needed) by passing the file name to dos2unix again via xargs. | |
| dos2unix --info=c0 php-support.md | tee have_crlf.txt | xargs -r0 dos2unix | |
| - name: Find generated section start/end markers in Dev Center article markdown | |
| id: find-section-markers | |
| run: | | |
| # init job file | |
| echo -n > php-support.md.ed-unordered.txt | |
| for f in section-*.md; do | |
| # extract first and last lines of the section file (those are the start and end markers) | |
| first=$(head -n1 "$f") | |
| last=$(tail -n1 "$f") | |
| # grep the line numbers (-n) as fixed (-F) full-line (-x) strings and extract them | |
| start=$(set -o pipefail; grep -nFx "$first" php-support.md | cut -d':' -f1) || { | |
| echo "::warning title=Failed to match section start marker::Start marker '$first' not found in input markdown; skipping '$f'..." | |
| continue | |
| } | |
| end=$(set -o pipefail; grep -nFx "$last" php-support.md | cut -d':' -f1) || { | |
| echo "::warning title=Failed to match section end marker::End marker '$last' not found in input markdown; skipping '$f'..." | |
| continue | |
| } | |
| # write out a line with the start-end range and filename | |
| echo "${start},${end} ${f}" >> php-support.md.ed-unordered.txt | |
| done | |
| num_sections=$(set -o pipefail; wc -l php-support.md.ed-unordered.txt | awk '{ print $1 }') | |
| (( $num_sections > 0 )) || echo "::warning title=No sections matched in input markdown::None of the generated sections coud be matched against the input markdown. No updates will occur." | |
| echo "num_sections=${num_sections}" >> "$GITHUB_OUTPUT" | |
| - name: Patch Dev Center article markdown | |
| if: steps.find-section-markers.outputs.num_sections > 0 | |
| run: | | |
| # init our ed script (https://www.gnu.org/software/diffutils/manual/html_node/Detailed-ed.html) for patching | |
| echo -n > php-support.md.ed | |
| # we now have the target file line ranges and source file names | |
| # for patch to handle the line numbers in the ed script correctly, they must be ordered with the last changes coming first | |
| # (otherwise every applied change will shift the line numbers for following changes) | |
| sort -r -n -k1 -t',' php-support.md.ed-unordered.txt | while read range f; do | |
| # write out an ed command that says "from starting line to ending line, replace with what follows" | |
| echo "${range}c" >> php-support.md.ed | |
| # write out new contents for range in command above | |
| cat "$f" >> php-support.md.ed | |
| # mark end of content | |
| echo "." >> php-support.md.ed | |
| done | |
| patch --backup --ed php-support.md php-support.md.ed | |
| - name: Dump diff of markdown contents | |
| id: diff | |
| if: steps.find-section-markers.outputs.num_sections > 0 | |
| run: | | |
| # diff exits 0 if there are no differences, 1 if there are, 2 if there was trouble | |
| # our patch --ed earlier added a newline at the end of the file | |
| # the downloaded markdown, however, may not have a newline at the end of the file | |
| # as a result, `diff` may produce a difference for that, but we want to ignore it | |
| # we run through a sed that matches on end of file ($), then appends (a) nothing (after backslash) | |
| # GNU sed will then add a newline at the end if there isn't one | |
| diff -u <(sed -e '$a\' php-support.md.orig) <(sed -e '$a\' php-support.md) > php-support.diff && { | |
| echo "::notice title=No diff in markdown::There were no differences after applying the generated sections to the input markdown." | |
| echo "diff_result=0" >> "$GITHUB_OUTPUT" | |
| } || { | |
| diff_result=$? | |
| echo "diff_result=${diff_result}" >> "$GITHUB_OUTPUT" | |
| (( diff_result != 1 )) && { | |
| echo "::error title=Unexpected error during diffing::Exit status of 'diff' command was '${diff_result}'." | |
| exit ${diff_result} | |
| } | |
| echo '## Diff of changes to ["PHP Support" Dev Center article](https://devcenter.heroku.com/articles/php-support)' >> "$GITHUB_STEP_SUMMARY" | |
| echo "${{ inputs.dry-run == true && '**This is based on the source bucket (due to dry-run mode)**, not the production bucket.' || '-n' }}" >> "$GITHUB_STEP_SUMMARY" | |
| echo '``````diff' >> "$GITHUB_STEP_SUMMARY" # six instead of three backticks because our output is likely to also contain series of backticks | |
| cat php-support.diff >> "$GITHUB_STEP_SUMMARY" | |
| echo '``````' >> "$GITHUB_STEP_SUMMARY" # six instead of three backticks because our output is likely to also contain series of backticks | |
| } | |
| - name: Upload diff as artifact | |
| if: steps.find-section-markers.outputs.num_sections > 0 && steps.diff.outputs.diff_result == 1 | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: devcenter-diff | |
| path: | | |
| have_crlf.txt | |
| php-support.diff | |
| devcenter-update: | |
| needs: devcenter-generate | |
| if: ${{ needs.devcenter-generate.outputs.diff_result == 1 }} | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install dos2unix | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install dos2unix | |
| - name: Download current Dev Center article markdown | |
| run: | | |
| set -o pipefail | |
| # we need to query this JSON twice, and a temp file is easiest for error handling | |
| curl -H "Accept: application/json" https://devcenter.heroku.com/api/v1/articles/php-support > php-support.json | |
| # jq -j, not -r, otherwise we get a stray trailing newline | |
| cat php-support.json | jq -je '.content' > php-support.md | |
| # here we want the trailing newline; run through alternative operator and empty filter to cause exit status 4 with -e if field missing | |
| cat php-support.json | jq -re '.id // empty | @sh "article_id=\(.)"' >> "$GITHUB_ENV" | |
| # Because the articles are edited in a web interface, they likely use CRLF line endings. | |
| # We will be patching using an LF diff. | |
| # For this reason, we may have to convert to LF, so we check if the file would be converted by dos2unix using the --info option. | |
| # The "c" info flag prints only file names that would trigger conversion; we first remember this output for the next step via tee -a. | |
| # The "0" flag triggers zero-byte output for happy consumption by xargs | |
| # Then, we finally run the conversion (if needed) by passing the file name to dos2unix again via xargs. | |
| dos2unix --info=c0 php-support.md | tee have_crlf.txt | xargs -r0 dos2unix | |
| - name: Download diff artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: devcenter-diff | |
| path: devcenter-diffs | |
| - name: Patch Dev Center article markdown | |
| run: | | |
| patch php-support.md devcenter-diffs/php-support.diff | |
| # convert back to the original CRLF if dos2unix ran in an earlier step (have_crlf.txt will be empty if not, and xargs will not run due to -r) | |
| cat have_crlf.txt | xargs -r0 unix2dos | |
| - name: Validate Dev Center article | |
| if: inputs.dry-run == true || inputs.update-devcenter == false | |
| run: | | |
| set -o pipefail | |
| echo curl -sS --fail-with-body -X POST -K <(echo -n "user = bot:"; printenv "HEROKU_DEVCENTER_API_TOKEN") -H "Accept: application/json" --data-urlencode 'article[content]@php-support.md' "https://devcenter.heroku.com/api/v1/private/articles/${article_id}/validate.json" | |
| - name: Output Dev Center article | |
| if: inputs.dry-run == true || inputs.update-devcenter == false | |
| run: | | |
| echo '## Updated markdown for ["PHP Support" Dev Center article](https://devcenter.heroku.com/articles/php-support)' >> "$GITHUB_STEP_SUMMARY" | |
| echo "${{ inputs.dry-run == true && '> [!IMPORTANT]' || '-n' }}" >> "$GITHUB_STEP_SUMMARY" | |
| echo "${{ inputs.dry-run == true && '**This is based on the source bucket (due to dry-run mode)**, not the production bucket.' || '-n' }}" >> "$GITHUB_STEP_SUMMARY" | |
| echo "${{ inputs.dry-run == true && '' || '-n' }}" >> "$GITHUB_STEP_SUMMARY" | |
| echo "> [!WARNING]" >> "$GITHUB_STEP_SUMMARY" | |
| echo "> **Dev Center has not been updated with the contents below**, the output is for reference only" >> "$GITHUB_STEP_SUMMARY" | |
| echo >> "$GITHUB_STEP_SUMMARY" | |
| echo '``````markdown' >> "$GITHUB_STEP_SUMMARY" # six instead of three backticks because our output is likely to also contain series of backticks | |
| cat php-support.md >> "$GITHUB_STEP_SUMMARY" | |
| echo '``````' >> "$GITHUB_STEP_SUMMARY" # six instead of three backticks because our output is likely to also contain series of backticks | |
| - name: Update Dev Center article | |
| if: inputs.dry-run == false && inputs.update-devcenter == true | |
| run: | | |
| set -o pipefail | |
| echo curl -sS --fail-with-body -X PUT -K <(echo -n "user = bot:"; printenv "HEROKU_DEVCENTER_API_TOKEN") -H "Accept: application/json" --data-urlencode 'article[content]@php-support.md' "https://devcenter.heroku.com/api/v1/private/articles/${article_id}.json" | |
| echo 'Successfully updated ["PHP Support" Dev Center article](https://devcenter.heroku.com/articles/php-support) with synced packages.' >> "$GITHUB_STEP_SUMMARY" | |
| changelog-generate: | |
| needs: sync | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install PHP and Composer | |
| uses: shivammathur/setup-php@v2 | |
| with: | |
| php-version: "8.2" | |
| tools: "composer:2.7" | |
| - name: Install Dev Center generator dependencies | |
| run: | | |
| composer install -d support/devcenter/ | |
| - name: Download all sync log artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: synclogs | |
| pattern: synclog-* | |
| merge-multiple: true | |
| - name: Generate Changelog markdown | |
| run: | | |
| set -o pipefail | |
| # concatenate all sync logs and feed them to changelog.php | |
| # immediately split changelog title and body (separated by a "----" line) for easier copy/paste | |
| # this also lets us detect if there even is anything to output in the next step (there won't be a changelog-01.md if the changelog is empty) | |
| # the '{*}' pattern works around a bug in older coreutils csplits: https://github.com/coreutils/coreutils/commit/7cf45f4f6a093a927d3139c87f52999dd2c750ec | |
| cat synclogs/sync-*.log | support/devcenter/changelog.php | csplit -s -z -f 'changelog-' -b '%02d.md' --suppress-matched - '/^----$/' '{*}' | |
| - name: Output Changelog markdown | |
| if: ${{ hashFiles('changelog-01.md') != '' }} | |
| run: | | |
| shopt -s extglob | |
| echo '## Markdown for package Changelog entry' >> "$GITHUB_STEP_SUMMARY" | |
| echo '### Title' >> "$GITHUB_STEP_SUMMARY" | |
| echo '``````markdown' >> "$GITHUB_STEP_SUMMARY" # six instead of three backticks because our output is likely to also contain series of backticks | |
| cat changelog-00.md >> "$GITHUB_STEP_SUMMARY" | |
| echo '``````' >> "$GITHUB_STEP_SUMMARY" # six instead of three backticks because our output is likely to also contain series of backticks | |
| echo '### Content' >> "$GITHUB_STEP_SUMMARY" | |
| echo '``````markdown' >> "$GITHUB_STEP_SUMMARY" # six instead of three backticks because our output is likely to also contain series of backticks | |
| cat changelog-!(00).md >> "$GITHUB_STEP_SUMMARY" | |
| echo '``````' >> "$GITHUB_STEP_SUMMARY" # six instead of three backticks because our output is likely to also contain series of backticks |