image: mono refs/heads/edge, buildroot - #3857
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 'Build OT2 image on github workflows' | |
| run-name: 'image: mono ${{ inputs.monorepo-ref }}, buildroot ${{ inputs.buildroot-ref }}' | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| monorepo-ref: | |
| description: | | |
| Ref of https://github.com/opentrons/opentrons to build. This MUST be a full ref, e.g. refs/heads/edge, or '-' to indicate not-specified. If not specified, will be determined from the oe-core ref if specified, and then default to edge. | |
| required: true | |
| default: '-' | |
| buildroot-ref: | |
| description: | | |
| Ref of https://github.com/opentrons/buildroot to build. This is different from the ref specified in the github api/webUI when starting this workflow - that ref is what contains this workflow, this ref specifies what gets built. It MUST be a full ref, e.g. refs/heads/main, or '-' to indicate not-specified. If not specified, will be decided based on the monorepo ref; if that isn't specified, will be main. | |
| required: true | |
| default: '-' | |
| infra-stage: | |
| description: | | |
| What infra stage to run on. This should almost always be prod; dev is useful when you explicitly want to test that stage. | |
| required: true | |
| type: choice | |
| options: | |
| - 'stage-prod' | |
| - 'stage-dev' | |
| default: 'stage-prod' | |
| env: | |
| CI: true | |
| jobs: | |
| decide-refs: | |
| name: 'Decide refs to build' | |
| runs-on: 'ubuntu-latest' | |
| outputs: | |
| buildroot: ${{ steps.build-refs.outputs.buildroot }} | |
| monorepo: ${{ steps.build-refs.outputs.monorepo }} | |
| variant: ${{ steps.build-refs.outputs.variant }} | |
| build-type: ${{ steps.build-refs.outputs.build-type }} | |
| steps: | |
| - name: Check out sources for action | |
| uses: 'actions/checkout@v3' | |
| with: | |
| path: ./buildroot-for-workflow | |
| - name: Get the refs | |
| id: build-refs | |
| uses: './buildroot-for-workflow/.github/actions/build-refs' | |
| with: | |
| token: ${{ github.token }} | |
| monorepo: ${{ inputs.monorepo-ref }} | |
| buildroot: ${{ inputs.buildroot-ref }} | |
| run-build: | |
| needs: decide-refs | |
| strategy: | |
| matrix: | |
| build_env: [ '${{ inputs.infra-stage }}', ] | |
| name: 'Building ${{needs.decide-refs.outputs.variant}} images on ${{ matrix.build_env }}' | |
| timeout-minutes: 480 | |
| runs-on: ['self-hosted', '${{matrix.build_env}}', '${{needs.decide-refs.outputs.variant}}'] | |
| concurrency: | |
| group: ${{needs.decide-refs.outputs.monorepo}} ${{needs.decide-refs.outputs.buildroot}} ${{needs.decide-refs.outputs.variant}} ${{needs.decide-refs.outputs.build-type}} ${{inputs.infra-stage}} | |
| cancel-in-progress: false | |
| steps: | |
| - name: Set up vm requirements | |
| run: | | |
| echo 'fs.inotify.max_user_watches=655360' | tee -a /etc/sysctl.conf | |
| echo 'fs.inotify.max_user_instances=1280' | tee -a /etc/sysctl.conf | |
| echo 'fs.file-max=100000' | tee -a /etc/sysctl.conf | |
| sysctl -p | |
| - name: Fetch initial sources for action | |
| uses: 'actions/checkout@v3' | |
| with: | |
| fetch-depth: 0 | |
| path: ./buildroot-for-workflow | |
| - name: Fetch buildroot source | |
| uses: 'actions/checkout@v3' | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{needs.decide-refs.outputs.buildroot}} | |
| path: ./buildroot | |
| - name: Fetch monorepo source | |
| uses: 'actions/checkout@v3' | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ needs.decide-refs.outputs.monorepo }} | |
| repository: Opentrons/opentrons | |
| path: ./opentrons | |
| - name: Configure AWS Credentials | |
| uses: './buildroot-for-workflow/.github/actions/aws-credentials' | |
| id: aws | |
| with: | |
| access_key_id: ${{ secrets.ROBOT_STACK_AWS_ACCESS_KEY_ID }} | |
| secret_access_key: ${{ secrets.ROBOT_STACK_AWS_SECRET_ACCESS_KEY }} | |
| region: us-east-2 | |
| stage: ${{ matrix.build_env }} | |
| - name: Pull or create cache locations | |
| id: cache-setup | |
| run: | | |
| cachedir=${LOCAL_CACHE:-./cache} | |
| for cachetype in downloads output ; do | |
| absdir="${cachedir}/${cachetype}" | |
| mkdir -p ${absdir} | |
| echo "${cachetype}=${absdir}" >> $GITHUB_OUTPUT | |
| here=$(pwd) | |
| reldir=$(realpath --relative-to=${here} ${cachedir}/${cachetype}) | |
| echo "Created ${cachetype} cache at ${absdir} (${reldir})" | |
| echo "${cachetype}-rel=${reldir}" >> $GITHUB_OUTPUT | |
| done | |
| - name: Set up docker environment file | |
| id: docker-env | |
| run: | | |
| cat <<EOF >./docker-envfile | |
| OT_BUILD_TYPE=${{needs.decide-refs.outputs.build-type}} | |
| FORCE_UNSAFE_CONFIGURE=1 | |
| BR2_DL_DIR=/downloads | |
| EOF | |
| echo "envfile=$(pwd)/docker-envfile" >> $GITHUB_OUTPUT | |
| - name: Build or download docker container | |
| id: get-image | |
| run: | | |
| cd buildroot | |
| imgname=$(./opentrons-build-container.sh pull || ./opentrons-build-container.sh build) | |
| echo "image-name=$imgname" >> $GITHUB_OUTPUT | |
| - name: Set up signing | |
| if: needs.decide-refs.outputs.build-type == 'release' | |
| run: | | |
| echo "${{secrets.ROBOT_SIGNING_KEY}}" > ./buildroot/.signing-key | |
| - name: Set up docker args | |
| id: docker-args | |
| run: | | |
| if [ ${{needs.decide-refs.outputs.variant}} = "release" ] ; then | |
| _project=robot-stack | |
| else | |
| _project=ot3 | |
| fi | |
| echo "Stack name ${_project} from variant ${{needs.decide-refs.outputs.variant}}" | |
| mount_args="type=bind,consistency=delegated" | |
| bind_br="--mount source=$(pwd)/buildroot,destination=/buildroot,${mount_args}" | |
| bind_ot="--mount source=$(pwd)/opentrons,destination=/opentrons,${mount_args}" | |
| bind_dl="--mount source=${{steps.cache-setup.outputs.downloads}},destination=/downloads,${mount_args}" | |
| bind_op="--mount source=${{steps.cache-setup.outputs.output}},destination=/output,${mount_args}" | |
| binds="${bind_br} ${bind_ot} ${bind_dl} ${bind_op}" | |
| env="--env-file ${{steps.docker-env.outputs.envfile}}" | |
| echo "run-binds=${binds}" >> $GITHUB_OUTPUT | |
| echo "env=${env}" >> $GITHUB_OUTPUT | |
| args="${binds} ${env} ${{steps.get-image.outputs.image-name}} O=/output OPENTRONS_PROJECT=${_project}" | |
| echo "args=${args}" >> $GITHUB_OUTPUT | |
| - name: Configure | |
| run: | | |
| cd buildroot | |
| docker run ${{steps.docker-args.outputs.args}} ot2_defconfig | |
| - name: Download package sources | |
| run: | | |
| cd buildroot | |
| docker run ${{steps.docker-args.outputs.args}} source | |
| - name: Run build | |
| run: | | |
| cd buildroot | |
| docker run ${{steps.docker-args.outputs.args}} all | |
| - name: Upload build log result | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: buildlog.txt | |
| path: ./buildroot/buildlog.txt | |
| - name: Prep artifacts for upload | |
| id: artifact-copy | |
| run: | | |
| artifact_dir="${{steps.cache-setup.outputs.output}}/images" | |
| prepped_artifact_dir="${{steps.cache-setup.outputs.output}}/unversioned_artifacts" | |
| mkdir -p ${prepped_artifact_dir} | |
| echo "prepped_artifact_dir=${prepped_artifact_dir}" >> $GITHUB_OUTPUT | |
| versioned_artifact_dir_rel="./versioned_artifacts" | |
| versioned_artifact_dir=$(realpath ${versioned_artifact_dir_rel}) | |
| mkdir -p ${versioned_artifact_dir} | |
| echo "versioned_artifact_dir=${versioned_artifact_dir}" >> $GITHUB_OUTPUT | |
| echo "versioned_artifact_dir_rel=${versioned_artifact_dir_rel}" >> $GITHUB_OUTPUT | |
| _fulltag="${{needs.decide-refs.outputs.monorepo}}" | |
| echo "monorepo_shorttag=${_fulltag:10}" >> $GITHUB_OUTPUT | |
| _vers_tag=${{needs.decide-refs.outputs.variant == 'release' && '${_fulltag:11}' || '${_fulltag:14}'}} | |
| cp ${artifact_dir}/ot2-system.zip ${prepped_artifact_dir}/ot2-system.zip | |
| echo "ot2_system=${prepped_artifact_dir}/ot2-system.zip" >> $GITHUB_OUTPUT | |
| cp ${artifact_dir}/ot2-fullimage.zip ${prepped_artifact_dir}/ot2-fullimage.zip | |
| echo "ot2_fullimage=${prepped_artifact_dir}/ot2-fullimage.zip" >> $GITHUB_OUTPUT | |
| cp ${artifact_dir}/VERSION.json ${prepped_artifact_dir}/VERSION.json | |
| echo "version_json=${prepped_artifact_dir}/VERSION.json" >> $GITHUB_OUTPUT | |
| cp ${artifact_dir}/release-notes.md ${prepped_artifact_dir}/release-notes.md | |
| echo "release_notes=${prepped_artifact_dir}/release-notes.md" >> $GITHUB_OUTPUT | |
| versioned_system_zip="ot2-system-${_vers_tag}.zip" | |
| cp ${artifact_dir}/ot2-system.zip "${versioned_artifact_dir}/${versioned_system_zip}" | |
| echo "ot2_system_versioned=${versioned_artifact_dir}/${versioned_system_zip}" >> $GITHUB_OUTPUT | |
| versioned_fullimage_zip="ot2-fullimage-${_vers_tag}.zip" | |
| cp ${artifact_dir}/ot2-fullimage.zip ${versioned_artifact_dir}/${versioned_fullimage_zip} | |
| echo "ot2_fullimage_versioned=${versioned_artifact_dir}/${versioned_fullimage_zip}" >> $GITHUB_OUTPUT | |
| versioned_version_json="VERSION-ot2-${_vers_tag}.json" | |
| cp ${artifact_dir}/VERSION.json ${versioned_artifact_dir}/${versioned_version_json} | |
| echo "version_json_versioned=${versioned_artifact_dir}/${versioned_version_json}" >> $GITHUB_OUTPUT | |
| echo "Prepared unversioned artifacts: $(find ${prepped_artifact_dir}/)" | |
| echo "Prepared versioned artifacts: $(find ${versioned_artifact_dir}/)" | |
| - name: Upload results to S3 | |
| shell: bash | |
| id: 'upload-results' | |
| run: | | |
| prefix=ot2-br | |
| arnslug=${S3_ARTIFACT_ARN/arn:aws:s3:::/} | |
| root_uri=${arnslug}/${prefix}/${{github.run_id}} | |
| pushd ${{steps.artifact-copy.outputs.prepped_artifact_dir}} | |
| aws --profile=${{ steps.aws.outputs.profile_name }} s3 cp --recursive --acl=public-read . s3://${root_uri}/ | |
| root_url=https://${root_uri} | |
| echo "console_url=https://s3.console.aws.amazon.com/s3/buckets/${arnslug}?prefix=${{github.run_id}}" >> $GITHUB_OUTPUT | |
| echo "version_file_url=$root_url/VERSION.json" >> $GITHUB_OUTPUT | |
| echo "release_notes_file_url=$root_url/release-notes.md" >> $GITHUB_OUTPUT | |
| echo "system_url=$root_url/ot2-system.zip" >> $GITHUB_OUTPUT | |
| echo "fullimage_url=$root_url/ot2-fullimage.zip" >> $GITHUB_OUTPUT | |
| echo "arnslug=$arnslug" >> $GITHUB_OUTPUT | |
| echo "root_uri=$root_uri" >> $GITHUB_OUTPUT | |
| echo "root_url=$root_url" >> $GITHUB_OUTPUT | |
| echo "root_prefix=ot2-br" >> $GITHUB_OUTPUT | |
| echo "machine_root=${arnslug}/${prefix}" >> $GITHUB_OUTPUT | |
| popd | |
| - name: Handle release manifest | |
| if: ${{ needs.decide-refs.outputs.build-type == 'release' }} | |
| shell: bash | |
| run: | | |
| pushd buildroot | |
| aws --profile=${{ steps.aws.outputs.profile_name }} s3 cp --acl=public-read s3://${{steps.upload-results.outputs.machine_root}}/releases.json releases.json | |
| python3 update_releases_file.py --releases-file releases.json --version-file ${{steps.artifact-copy.outputs.version_json}} --base-url ${{ steps.upload-results.outputs.root_url }} | |
| aws --profile=${{ steps.aws.outputs.profile_name }} s3 cp --acl=public-read releases.json s3://${{steps.upload-results.outputs.machine_root}}/releases.json | |
| popd | |
| - name: Upload system zip to monorepo release | |
| if: needs.decide-refs.outputs.build-type == 'release' | |
| uses: 'ncipollo/[email protected]' | |
| with: | |
| allowUpdates: true | |
| omitBody: true | |
| omitName: true | |
| omitPrereleaseDuringUpdate: true | |
| omitDraftDuringUpdate: true | |
| repo: opentrons | |
| tag: ${{ steps.artifact-copy.outputs.monorepo_shorttag }} | |
| artifacts: "${{steps.artifact-copy.outputs.versioned_artifact_dir_rel}}/ot2-*.zip" | |
| artifactContentType: application/zip | |
| token: ${{secrets.MONOREPO_RELEASE_ARTIFACT_UPLOAD_TOKEN}} | |
| - name: Upload version json to monorepo release | |
| if: needs.decide-refs.outputs.build-type == 'release' | |
| uses: 'ncipollo/[email protected]' | |
| with: | |
| allowUpdates: true | |
| omitBody: true | |
| omitName: true | |
| omitPrereleaseDuringUpdate: true | |
| omitDraftDuringUpdate: true | |
| repo: opentrons | |
| tag: ${{ steps.artifact-copy.outputs.monorepo_shorttag }} | |
| artifacts: "${{steps.artifact-copy.outputs.versioned_artifact_dir_rel}}/VERSION-*.json" | |
| artifactContentType: application/json | |
| token: ${{secrets.MONOREPO_RELEASE_ARTIFACT_UPLOAD_TOKEN}} | |
| - name: Post results as internal-release | |
| if: matrix.build_env == 'stage-prod' && needs.decide-refs.outputs.variant == 'internal-release' | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: "{\"s3-url\":\"${{ steps.upload-results.outputs.console_url }}/\",\"type\":\"branch\", \"reflike\":\"${{ needs.decide-refs.outputs.buildroot }}\",\"monorepo-reflike\":\"${{ needs.decide-refs.outputs.monorepo }}\",\"full-image\":\"${{ steps.upload-results.outputs.fullimage_url }}\", \"system-update\":\"${{ steps.upload-results.outputs.system_url }}\", \"version-file\":\"${{ steps.upload-results.outputs.version_file_url }}\", \"release-notes\":\"${{ steps.upload-results.outputs.release_notes_file_url }}\"}" | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_INTERNAL_RELEASE }} | |
| - name: Post results as release | |
| if: matrix.build_env == 'stage-prod' && needs.decide-refs.outputs.variant == 'release' | |
| uses: slackapi/[email protected] | |
| with: | |
| payload: "{\"s3-url\":\"${{ steps.upload-results.outputs.console_url }}/\",\"type\":\"branch\", \"reflike\":\"${{ needs.decide-refs.outputs.buildroot }}\",\"monorepo-reflike\":\"${{ needs.decide-refs.outputs.monorepo }}\",\"full-image\":\"${{ steps.upload-results.outputs.fullimage_url }}\", \"system-update\":\"${{ steps.upload-results.outputs.system_url }}\", \"version-file\":\"${{ steps.upload-results.outputs.version_file_url }}\", \"release-notes\":\"${{ steps.upload-results.outputs.release_notes_file_url }}\"}" | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_RELEASE }} | |
| - name: remove build data | |
| if: always() | |
| run: | | |
| rm -rf ./* | |
| output_check=${{steps.cache-setup.outputs.output}} | |
| rm -rf ${output_check:-/does/not/exist} |