From 16884c13dc219a6991689f78d63915e21b227b68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Ulus?= Date: Mon, 22 Jul 2024 18:23:05 +0300 Subject: [PATCH] Add a workflow using buildah (#4) --- .github/workflows/builder-build.yml | 38 +------- .github/workflows/builder-buildah-build.yml | 103 ++++++++++++++++++++ Dockerfile | 26 +++-- autoware.repos.yml | 16 +-- 4 files changed, 134 insertions(+), 49 deletions(-) create mode 100644 .github/workflows/builder-buildah-build.yml diff --git a/.github/workflows/builder-build.yml b/.github/workflows/builder-build.yml index 1ec8d17..c47c359 100644 --- a/.github/workflows/builder-build.yml +++ b/.github/workflows/builder-build.yml @@ -1,8 +1,6 @@ -name: Build Autoware builder container images +name: Build Autoware builder container images (Docker) on: workflow_dispatch: - schedule: - - cron: '34 15 * * 0' # 15:34 UTC every Sunday push: paths: - Dockerfile @@ -10,8 +8,8 @@ on: - .github/workflows/builder-build.yml # Self-trigger env: - REGISTRY: ghcr.io - IMAGE_NAME: bounverif/autoware + REGISTRY: ghcr.io/bounverif + IMAGE_NAME: autoware AUTOWARE_VERSION: latest concurrency: @@ -79,33 +77,3 @@ jobs: cache-from: | type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.AUTOWARE_VERSION }}-builder cache-to: type=inline - - - name: Push builder image - uses: docker/build-push-action@v6 - with: - context: . - target: autoware-builder - build-args: | - "AUTOWARE_VERSION=${{ env.AUTOWARE_VERSION }}" - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-builder - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.AUTOWARE_VERSION }}-builder - push: true - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.AUTOWARE_VERSION }}-builder - cache-to: type=inline - - - name: Push builder-with-cache image - uses: docker/build-push-action@v6 - with: - context: . - target: autoware-builder-with-cache - build-args: | - "AUTOWARE_VERSION=${{ env.AUTOWARE_VERSION }}" - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-builder-with-cache - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.AUTOWARE_VERSION }}-builder-with-cache - push: true - cache-from: | - type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.AUTOWARE_VERSION }}-builder - type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.AUTOWARE_VERSION }}-builder-with-cache - cache-to: type=inline diff --git a/.github/workflows/builder-buildah-build.yml b/.github/workflows/builder-buildah-build.yml new file mode 100644 index 0000000..93b076e --- /dev/null +++ b/.github/workflows/builder-buildah-build.yml @@ -0,0 +1,103 @@ +name: Build Autoware builder container images using Buildah +on: + workflow_dispatch: + schedule: + - cron: '34 15 * * 0' # 15:34 UTC every Sunday + push: + paths: + - Dockerfile + - autoware.repos.yml + - .github/workflows/builder-buildah-build.yml # Self-trigger + +env: + REGISTRY: ghcr.io/bounverif + IMAGE_NAME: autoware + AUTOWARE_VERSION: latest + CONTAINERS_ROOT: /home/runner/.local/share/containers + TMPDIR: /home/runner/.local/share/containers/tmp + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + packages: write + +jobs: + autoware-builder-build: + runs-on: ubuntu-24.04 + steps: + - name: Maximize build space + uses: easimon/maximize-build-space@v10 + with: + root-reserve-mb: 2048 # Reserve disk space for repository + remove-dotnet: 'true' + remove-android: 'true' + remove-haskell: 'true' + remove-codeql: 'true' + remove-docker-images: 'true' + build-mount-path: ${{ env.CONTAINERS_ROOT }} # The remaining space only for container build + + - run: mkdir -p $TMPDIR + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set current date as the version + run: echo "AUTOWARE_VERSION=$(date +'%Y%m%d')" >> $GITHUB_ENV + + - name: Build builder container image + id: build-image + uses: redhat-actions/buildah-build@v2 + with: + platforms: linux/amd64 + image: ${{ env.IMAGE_NAME }} + tags: ${{ env.AUTOWARE_VERSION }}-builder latest-builder + oci: true + build-args: | + AUTOWARE_VERSION=${{ env.AUTOWARE_VERSION }} + extra-args: + --target autoware-builder + --cache-from=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + --cache-to=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + --cache-ttl=24h + containerfiles: | + ./Dockerfile + + - name: Build builder-with-cache container image + id: build-builder-with-cache + uses: redhat-actions/buildah-build@v2 + with: + platforms: linux/amd64 + image: ${{ env.IMAGE_NAME }} + tags: ${{ env.AUTOWARE_VERSION }}-builder-with-cache latest-with-cache + oci: true + build-args: | + AUTOWARE_VERSION=${{ env.AUTOWARE_VERSION }} + extra-args: + --target autoware-builder-with-cache + --cache-from=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + --cache-to=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + containerfiles: | + ./Dockerfile + + - name: Log in to the GitHub Container registry + uses: redhat-actions/podman-login@v1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Push to GitHub Container Repository + id: push-builder-ghcr + uses: redhat-actions/push-to-registry@v2 + with: + registry: ${{ env.REGISTRY }} + image: ${{ steps.build-image.outputs.image }} + tags: ${{ steps.build-image.outputs.tags }} + extra-args: | + + - name: Print image url + run: echo "Image pushed to ${{ steps.push-builder-ghcr.outputs.registry-paths }}" + \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 2195ef4..1b9dcf2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,9 @@ -FROM ubuntu:22.04 AS autoware-base +FROM docker.io/library/ubuntu:22.04 AS autoware-base + +ARG TARGETARCH=amd64 +ARG TARGETOS=linux +ARG TARGETPLATFORM=linux/amd64 -ARG TARGETARCH ENV ID=ubuntu ENV VERSION_ID=22.04 ENV CACHEMOUNT_PREFIX=/${TARGETARCH}/${ID}${VERSION_ID} @@ -33,7 +36,9 @@ ARG CUDA_KEYRING_FILEPATH=https://developer.download.nvidia.com/compute/cuda/rep RUN rm -f /etc/apt/apt.conf.d/docker-clean RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${CACHEMOUNT_PREFIX}/var/cache/apt \ - apt-get update && apt-get install -y --no-install-recommends \ + export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -qy --no-install-recommends \ sudo \ tini \ wget \ @@ -61,7 +66,9 @@ RUN wget -qO- "https://keyserver.ubuntu.com/pks/lookup?fingerprint=on&op=get&sea FROM autoware-base AS autoware-source RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${CACHEMOUNT_PREFIX}/var/cache/apt \ - apt-get update && apt-get install -y --no-install-recommends \ + export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ git \ python3-minimal \ python3-vcstool \ @@ -74,7 +81,9 @@ RUN mkdir -p ${AUTOWARE_SOURCE_DIR} && vcs import --shallow ${AUTOWARE_SOURCE_DI FROM autoware-base AS autoware-builder-nocuda RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${CACHEMOUNT_PREFIX}/var/cache/apt \ - apt-get update && apt-get install -y --no-install-recommends \ + export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ build-essential \ git \ cmake \ @@ -89,7 +98,10 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${CACHEMOUNT_PREF RUN --mount=type=bind,from=autoware-source,source=${AUTOWARE_SOURCE_DIR},target=${AUTOWARE_SOURCE_DIR} \ --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${CACHEMOUNT_PREFIX}/var/cache/apt \ - rosdep init && apt update && rosdep update && \ + export DEBIAN_FRONTEND=noninteractive && \ + apt update && \ + rosdep init && \ + rosdep update && \ rosdep install -y \ --from-paths ${AUTOWARE_SOURCE_DIR} \ --ignore-src \ @@ -108,6 +120,7 @@ RUN mkdir -p ${CCACHE_DIR} && chmod 777 ${CCACHE_DIR} FROM autoware-builder-nocuda AS autoware-builder RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${CACHEMOUNT_PREFIX}/var/cache/apt \ + export DEBIAN_FRONTEND=noninteractive && \ apt-get update && apt-get install -y --no-install-recommends \ cuda-minimal-build-12-4 \ libcublas-dev-12-4 \ @@ -190,6 +203,7 @@ ENV NVIDIA_VISIBLE_DEVICES=all ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility,graphics RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${CACHEMOUNT_PREFIX}/var/cache/apt \ + export DEBIAN_FRONTEND=noninteractive && \ apt-get update && apt-get install -y --no-install-recommends \ libcublas-12-4 \ libcurand-12-4 \ diff --git a/autoware.repos.yml b/autoware.repos.yml index 961a50e..1ebbeb8 100644 --- a/autoware.repos.yml +++ b/autoware.repos.yml @@ -113,14 +113,14 @@ repositories: type: git url: https://github.com/autowarefoundation/sample_sensor_kit_launch.git version: main - sensor_kit/external/awsim_sensor_kit_launch: # TODO: Integrate into sample_sensor_kit_launch - type: git - url: https://github.com/RobotecAI/awsim_sensor_kit_launch.git - version: main - sensor_kit/awsim_labs_sensor_kit_launch: - type: git - url: https://github.com/autowarefoundation/awsim_labs_sensor_kit_launch.git - version: main + # sensor_kit/external/awsim_sensor_kit_launch: # TODO: Integrate into sample_sensor_kit_launch + # type: git + # url: https://github.com/RobotecAI/awsim_sensor_kit_launch.git + # version: main + # sensor_kit/awsim_labs_sensor_kit_launch: + # type: git + # url: https://github.com/autowarefoundation/awsim_labs_sensor_kit_launch.git + # version: main # vehicle vehicle/sample_vehicle_launch: type: git