Merge pull request #347 from buildkite-plugins/toote_buildkit_issue-256

BuildKit & `ssh` support

README.md

@@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using
 steps:
   - command: test.sh
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           run: app
 ```
 
@@ -28,7 +28,7 @@ through if you need:
 steps:
   - command: test.sh
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           run: app
           config: docker-compose.tests.yml
           env:
@@ -41,7 +41,7 @@ or multiple config files:
 steps:
   - command: test.sh
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           run: app
           config:
             - docker-compose.yml
@@ -56,7 +56,7 @@ env:
 steps:
   - command: test.sh
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           run: app
 ```
 
@@ -65,7 +65,7 @@ If you want to control how your command is passed to docker-compose, you can use
 ```yml
 steps:
   - plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           run: app
           command: ["custom", "command", "values"]
 ```
@@ -79,15 +79,15 @@ steps:
   - plugins:
       - docker-login#v2.0.1:
           username: xyz
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           build: app
           image-repository: index.docker.io/myorg/myrepo
   - wait
   - command: test.sh
     plugins:
       - docker-login#v2.0.1:
           username: xyz
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           run: app
 ```
 
@@ -104,7 +104,7 @@ steps:
   - command: generate-dist.sh
     artifact_paths: "dist/*"
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           run: app
 ```
 
@@ -122,7 +122,7 @@ steps:
   - command: generate-dist.sh
     artifact_paths: "dist/*"
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           run: app
           volumes:
             - "./dist:/app/dist"
@@ -146,7 +146,7 @@ this plugin offers a `environment` block of its own:
 steps:
   - command: generate-dist.sh
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           run: app
           env:
             - BUILDKITE_BUILD_NUMBER
@@ -164,7 +164,7 @@ Alternatively, you can have the plugin add all environment variables defined for
 steps:
   - command: use-vars.sh
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           run: app
           propagate-environment: true
 ```
@@ -179,7 +179,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th
 steps:
   - command: generate-dist.sh
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           build: app
           image-repository: index.docker.io/myorg/myrepo
           args:
@@ -196,7 +196,7 @@ If you have multiple steps that use the same service/image (such as steps that r
 steps:
   - label: ":docker: Build"
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           build: app
           image-repository: index.docker.io/myorg/myrepo
 
@@ -206,7 +206,7 @@ steps:
     command: test.sh
     parallelism: 25
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           run: app
 ```
 
@@ -222,7 +222,7 @@ steps:
     agents:
       queue: docker-builder
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           build:
             - app
             - tests
@@ -234,7 +234,7 @@ steps:
     command: test.sh
     parallelism: 25
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           run: tests
 ```
 
@@ -246,7 +246,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu
 steps:
   - label: ":docker: Push"
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           push: app
 ```
 
@@ -256,7 +256,7 @@ To push multiple images, you can use a list:
 steps:
   - label: ":docker: Push"
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           push:
             - first-service
             - second-service
@@ -268,7 +268,7 @@ If you want to push to a specific location (that's not defined as the `image` in
 steps:
   - label: ":docker: Push"
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           push:
             - app:index.docker.io/myorg/myrepo/myapp
             - app:index.docker.io/myorg/myrepo/myapp:latest
@@ -282,14 +282,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w
 steps:
   - label: ":docker: Build an image"
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           build: app
           image-repository: index.docker.io/myorg/myrepo
           cache-from: app:index.docker.io/myorg/myrepo/myapp:latest
   - wait
   - label: ":docker: Push to final repository"
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           push:
             - app:index.docker.io/myorg/myrepo/myapp
             - app:index.docker.io/myorg/myrepo/myapp:latest
@@ -303,7 +303,7 @@ This plugin allows for the value of `cache-from` to be a string or a list. If it
 steps:
   - label: ":docker Build an image"
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           build: app
           image-repository: index.docker.io/myorg/myrepo
           cache-from:
@@ -312,7 +312,7 @@ steps:
   - wait
   - label: ":docker: Push to final repository"
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           push:
             - app:index.docker.io/myorg/myrepo/myapp
             - app:index.docker.io/myorg/myrepo/myapp:my-branch
@@ -326,7 +326,7 @@ Adding a grouping tag to the end of a cache-from list item allows this plugin to
 steps:
   - label: ":docker: Build Intermediate Image"
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           build: myservice_intermediate  # docker-compose.yml is the same as myservice but has `target: intermediate`
           image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER}
           image-repository: index.docker.io/myorg/myrepo/myservice_intermediate
@@ -336,7 +336,7 @@ steps:
   - wait
   - label: ":docker: Build Final Image"
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           build: myservice
           image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER}
           image-repository: index.docker.io/myorg/myrepo
@@ -380,7 +380,7 @@ A basic pipeline similar to the following:
 steps:
   - label: ":docker: Run & Push"
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           run: myservice
           push: myservice
 ```
@@ -395,7 +395,7 @@ A basic pipeline similar to the following:
 steps:
   - label: ":docker: Build & Push"
     plugins:
-      - docker-compose#v4.1.1:
+      - docker-compose#v4.3.0:
           build: myservice
           push: myservice
 ```
@@ -612,6 +612,16 @@ To run the tests:
 docker-compose run --rm tests bats tests tests/v2
 ```
 
+### `buildkit` (optional, build only, boolean)
+
+Assuming you have a compatible docker installation and configuration in the agent, activating this option would setup the environment for the `docker-compose build` call to use BuildKit. Note that if you are using `cli-version` 2, you are already using buildkit by default.
+
+You may want to also add `BUILDKIT_INLINE_CACHE=1` to your build arguments (`args` option in this plugin), but know that [there are known issues with it](https://github.com/moby/buildkit/issues/2274).
+
+### `ssh` (optional, build only, boolean)
+
+When enabled, it will add the `--ssh` option to the build command. Note that it assumes you have a compatible docker installation and configuration in the agent (meaning you are using BuildKit and it is correctly setup).
+
 ## License
 
 MIT (see [LICENSE](LICENSE))

commands/build.sh

@@ -44,6 +44,12 @@ if [[ -z "$image_repository" ]] ; then
   echo "This build step has no image-repository set. Without an image-repository, the Docker image won't be pushed to a repository, and won't be automatically used by any run steps."
 fi
 
+if [[ "$(plugin_read_config BUILDKIT "false")" == "true" ]]; then
+  export DOCKER_BUILDKIT=1
+  export COMPOSE_DOCKER_CLI_BUILD=1
+  export BUILDKIT_PROGRESS=plain
+fi
+
 # Read any cache-from parameters provided and pull down those images first
 # If no-cache is set skip pulling the cache-from images
 if [[ "$(plugin_read_config NO_CACHE "false")" == "false" ]] ; then
@@ -137,7 +143,7 @@ while read -r line ; do
   [[ -n "$line" ]] && services+=("$line")
 done <<< "$(plugin_read_list BUILD)"
 
-build_params=(--pull)
+build_params=(build --pull)
 
 if [[ "$(plugin_read_config NO_CACHE "false")" == "true" ]] ; then
   build_params+=(--no-cache)
@@ -147,6 +153,14 @@ if [[ "$(plugin_read_config BUILD_PARALLEL "false")" == "true" ]] ; then
   build_params+=(--parallel)
 fi
 
+if [[ "$(plugin_read_config SSH "false")" == "true" ]] ; then
+  if [[ "${DOCKER_BUILDKIT:-}" != "1" && "${BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLI_VERSION:-}" != "2" ]]; then
+    echo "🚨 You can not use the ssh option if you are not using buildkit"
+    exit 1
+  fi
+  build_params+=(--ssh)
+fi
+
 target="$(plugin_read_config TARGET "")"
 if [[ -n "$target" ]] ; then
   build_params+=(--target "$target")
@@ -157,7 +171,7 @@ while read -r arg ; do
 done <<< "$(plugin_read_list ARGS)"
 
 echo "+++ :docker: Building services ${services[*]}"
-run_docker_compose -f "$override_file" build "${build_params[@]}" "${services[@]}"
+run_docker_compose -f "$override_file" "${build_params[@]}" "${services[@]}"
 
 if [[ -n "$image_repository" ]] ; then
   echo "~~~ :docker: Pushing built images to $image_repository"

plugin.yml

@@ -22,6 +22,8 @@ configuration:
     build-alias:
       type: [ string, array ]
       minimum: 1
+    buildkit:
+      type: boolean
     cache-from:
       type: [ string, array ]
       minimum: 1
@@ -76,6 +78,8 @@ configuration:
       type: boolean
     skip-pull:
       type: boolean
+    ssh:
+      type: boolean
     target:
       type: string
     tty:
@@ -101,6 +105,7 @@ configuration:
   additionalProperties: false
   dependencies:
     ansi: [ run ]
+    buildkit: [ build ]
     cache-from: [ build ]
     dependencies: [ run ]
     env: [ run ]
@@ -114,6 +119,7 @@ configuration:
     pull: [ run ]
     push-retries: [ push ]
     skip-pull: [ run ]
+    ssh: [ buildkit ]
     target: [ build ]
     tty: [ run ]
     use-aliases: [ run ]

tests/build.bats

@@ -732,5 +732,41 @@ load '../lib/shared'
   assert_output --partial "built myservice"
   assert_output --partial "with target intermediate"
 
+  unstub docker-compose
+}
+
+@test "Build with ssh option (but no buildkit)" {
+  export BUILDKITE_BUILD_NUMBER=1
+  export BUILDKITE_JOB_ID=1111
+  export BUILDKITE_PIPELINE_SLUG=test
+
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SSH=true
+
+  run "$PWD"/hooks/command
+
+  assert_failure
+  assert_output --partial "You can not use the ssh option if you are not using buildkit"
+  refute_output --partial "built myservice"
+}
+
+@test "Build with ssh option and buildkit" {
+  export BUILDKITE_BUILD_NUMBER=1
+  export BUILDKITE_JOB_ID=1111
+  export BUILDKITE_PIPELINE_SLUG=test
+
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILDKIT=true
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_SSH=true
+
+  stub docker-compose \
+    "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull --ssh \* : echo built \${10} with ssh"
+
+  run "$PWD"/hooks/command
+
+  assert_success
+  assert_output --partial "built myservice"
+  assert_output --partial "with ssh"
+
   unstub docker-compose
 }