From dd0a3f41f2f3d362e68bc9e0924bf718b8f6bff5 Mon Sep 17 00:00:00 2001 From: Jake Barnes Date: Mon, 11 Nov 2019 14:52:19 +1100 Subject: [PATCH 01/14] Remove restriction on build and run --- hooks/command | 8 -------- 1 file changed, 8 deletions(-) diff --git a/hooks/command b/hooks/command index 8e0692b1..41c857ca 100755 --- a/hooks/command +++ b/hooks/command @@ -12,14 +12,6 @@ commands=() [[ -n "$(plugin_read_list BUILD)" ]] && commands+=("BUILD") [[ -n "$(plugin_read_list RUN)" ]] && commands+=("RUN") - -# Check we've only got one of BUILD or RUN -if [[ ${#commands[@]} -gt 1 ]] ; then - echo "+++ Docker Compose plugin error" - echo "Only one of build or run is supported. More than one was used." - exit 1 -fi - [[ -n "$(plugin_read_list PUSH)" ]] && commands+=("PUSH") # Don't convert paths on gitbash on windows From 810c5afe38580d3140ebb884f052925cbb485ba3 Mon Sep 17 00:00:00 2001 From: Jeremy Bumsted Date: Tue, 12 Apr 2022 12:18:16 -0700 Subject: [PATCH 02/14] WIP - add tests for running multiple commands - tests pass, but fail when trying to unstub commands --- tests/multiple-commands.bats | 46 ++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tests/multiple-commands.bats diff --git a/tests/multiple-commands.bats b/tests/multiple-commands.bats new file mode 100644 index 00000000..edf4f4b4 --- /dev/null +++ b/tests/multiple-commands.bats @@ -0,0 +1,46 @@ +#!/usr/bin/env bats + +load '/usr/local/lib/bats/load.bash' +load '../lib/shared' + + export DOCKER_COMPOSE_STUB_DEBUG=/dev/stdout + export BUILDKITE_AGENT_STUB_DEBUG=/dev/stdout +# export BATS_MOCK_TMPDIR=$PWD + + +@test "Build and run in a single step" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND="echo hello world" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml push myservice : echo pushed myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --rm myservice /bin/sh -e -c echo hello world : echo ran myservice" + + stub buildkite-agent \ + "meta-data set docker-compose-plugin-built-image-tag-myservice my.repository/llamas:test-myservice-build-1 : echo set meta-data" \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : echo meta-data exists" + + run $PWD/hooks/command + + assert_success + assert_output --partial "built myservice" + assert_output --partial "ran myservice" + + #unstub docker-compose + #unstub buildkite-agent +} + + + + + From acdf35db377757da4da47155c8e4e1d7c32faf06 Mon Sep 17 00:00:00 2001 From: Jeremy Bumsted Date: Mon, 25 Apr 2022 15:18:36 -0700 Subject: [PATCH 03/14] fix README version numbers --- README.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index ed28d68a..a0d4c59a 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: run: app ``` @@ -26,7 +26,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: run: app config: docker-compose.tests.yml env: @@ -39,7 +39,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: run: app config: - docker-compose.yml @@ -53,7 +53,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -61,7 +61,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: run: app ``` @@ -70,7 +70,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: run: app command: ["custom", "command", "values"] ``` @@ -86,7 +86,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: run: app volumes: - "./dist:/app/dist" @@ -125,7 +125,7 @@ this plugin offers a `environment` block of it's own: steps: - command: generate-dist.sh plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -145,7 +145,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -162,7 +162,7 @@ To speed up run steps that use the same service/image (such as steps that run in steps: - label: ":docker: Build" plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -172,7 +172,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: run: app ``` @@ -188,7 +188,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: build: - app - tests @@ -200,7 +200,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: run: tests ``` @@ -212,7 +212,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: push: app ``` @@ -224,7 +224,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: push: app ``` @@ -236,7 +236,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: push: - first-service - second-service @@ -250,7 +250,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -264,14 +264,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker Build an image" plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v3.1.0: + - docker-compose#v3.9.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest From 9c5bca1387d67cb36c32c51253f5364880dea67d Mon Sep 17 00:00:00 2001 From: Jeremy Bumsted Date: Wed, 11 May 2022 18:23:09 -0700 Subject: [PATCH 04/14] fix tests --- tests/multiple-commands.bats | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/tests/multiple-commands.bats b/tests/multiple-commands.bats index edf4f4b4..e88c8b2f 100644 --- a/tests/multiple-commands.bats +++ b/tests/multiple-commands.bats @@ -2,12 +2,12 @@ load '/usr/local/lib/bats/load.bash' load '../lib/shared' +load '../lib/metadata' - export DOCKER_COMPOSE_STUB_DEBUG=/dev/stdout - export BUILDKITE_AGENT_STUB_DEBUG=/dev/stdout +# export DOCKER_COMPOSE_STUB_DEBUG=/dev/tty +# export BUILDKITE_AGENT_STUB_DEBUG=/dev/tty # export BATS_MOCK_TMPDIR=$PWD - @test "Build and run in a single step" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice @@ -20,27 +20,29 @@ load '../lib/shared' export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false stub docker-compose \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml push myservice : echo pushed myservice" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --rm myservice /bin/sh -e -c echo hello world : echo ran myservice" + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml push myservice : echo pushed myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" stub buildkite-agent \ - "meta-data set docker-compose-plugin-built-image-tag-myservice my.repository/llamas:test-myservice-build-1 : echo set meta-data" \ - "meta-data exists docker-compose-plugin-built-image-tag-myservice : echo meta-data exists" + "meta-data set docker-compose-plugin-built-image-tag-myservice my.repository/llamas:test-myservice-build-1 : echo set meta-data" \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo got meta-data" run $PWD/hooks/command + unstub docker-compose + unstub buildkite-agent + assert_success assert_output --partial "built myservice" - assert_output --partial "ran myservice" - - #unstub docker-compose - #unstub buildkite-agent + assert_output --partial "pushed myservice" + assert_output --partial "pulled myservice" + assert_output --partial "ran myservice dependencies" + assert_output --partial "ran myservice" } - - From 3b2933725432dd4fe8a62446e5c39fde2969964b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 16 Sep 2022 21:36:12 -0300 Subject: [PATCH 05/14] allow build and run at the same time --- plugin.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/plugin.yml b/plugin.yml index b0a1d41f..93ae0e4d 100644 --- a/plugin.yml +++ b/plugin.yml @@ -79,10 +79,11 @@ configuration: entrypoint: type: string oneOf: - - required: - - run - - required: - - build + - anyOf: + - required: + - run + - required: + - build - required: - push additionalProperties: false From fa62f5009915c339eb7212dc2da5e4a78b924960 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 16 Sep 2022 21:43:08 -0300 Subject: [PATCH 06/14] Add example in documentation about build & run --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 649eee60..96dc36e8 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,19 @@ steps: run: app ``` +Alternatively, you can do everything in a single step: +```yml +steps: + - command: test.sh + plugins: + - docker-login#v2.0.1: + username: xyz + - docker-compose#v3.11.0: + build: app + image-repository: index.docker.io/myorg/myrepo + run: app +``` + If you want to control how your command is passed to docker-compose, you can use the command parameter on the plugin directly: ```yml From a3b35d1fe23265d9f70e7ccae408052aacad45e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Fri, 16 Sep 2022 21:50:41 -0300 Subject: [PATCH 07/14] pin plugin tester to older version --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8023abd9..fe299c6b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ version: '2' services: tests: - image: buildkite/plugin-tester + image: buildkite/plugin-tester:v2.0.0 volumes: - ".:/plugin" From c88e071e6d13a5c9f1c7a061bd5d3ab61c28913d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sun, 25 Sep 2022 20:58:59 -0300 Subject: [PATCH 08/14] Simplified multiple-commands test file --- tests/multiple-commands.bats | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/multiple-commands.bats b/tests/multiple-commands.bats index e88c8b2f..9224cd09 100644 --- a/tests/multiple-commands.bats +++ b/tests/multiple-commands.bats @@ -8,23 +8,26 @@ load '../lib/metadata' # export BUILDKITE_AGENT_STUB_DEBUG=/dev/tty # export BATS_MOCK_TMPDIR=$PWD +# General pipeline variables +export BUILDKITE_BUILD_NUMBER=1 +export BUILDKITE_COMMAND="pwd" +export BUILDKITE_JOB_ID=12 +export BUILDKITE_PIPELINE_SLUG=test + + @test "Build and run in a single step" { - export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice - export BUILDKITE_PIPELINE_SLUG=test - export BUILDKITE_BUILD_NUMBER=1 - export BUILDKITE_COMMAND="echo hello world" export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false stub docker-compose \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml push myservice : echo pushed myservice" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran myservice dependencies" \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice" + "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" \ + "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml push myservice : echo pushed myservice" \ + "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran dependencies" \ + "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml run --name buildkite12_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" stub buildkite-agent \ "meta-data set docker-compose-plugin-built-image-tag-myservice my.repository/llamas:test-myservice-build-1 : echo set meta-data" \ @@ -40,7 +43,7 @@ load '../lib/metadata' assert_output --partial "built myservice" assert_output --partial "pushed myservice" assert_output --partial "pulled myservice" - assert_output --partial "ran myservice dependencies" + assert_output --partial "ran dependencies" assert_output --partial "ran myservice" } From 29bacf53a19d6c8f8128c25d5add8018fc3fb8c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sun, 25 Sep 2022 21:00:06 -0300 Subject: [PATCH 09/14] It is now possible to run any combination of commands --- plugin.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/plugin.yml b/plugin.yml index 93ae0e4d..1086916a 100644 --- a/plugin.yml +++ b/plugin.yml @@ -78,12 +78,11 @@ configuration: type: boolean entrypoint: type: string - oneOf: - - anyOf: - - required: - - run - - required: - - build + anyOf: + - required: + - run + - required: + - build - required: - push additionalProperties: false From c62227ebcfd5fe4978dad4ba17b9637b274ee5ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sun, 25 Sep 2022 21:15:23 -0300 Subject: [PATCH 10/14] Did a read-through of the documentation --- README.md | 54 +++++++++++++++++++++--------------------------------- 1 file changed, 21 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 8765d1a9..15d914ea 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,18 @@ steps: run: app ``` +If you want to control how your command is passed to docker-compose, you can use the command parameter on the plugin directly: + +```yml +steps: + - plugins: + - docker-compose#v3.11.2: + run: app + command: ["custom", "command", "values"] +``` + +## Authenticated registries + You can leverage the [docker-login plugin](https://github.com/buildkite-plugins/docker-login-buildkite-plugin) in tandem for authenticating with a registry. For example, the following will build and push an image to a private repo, and pull from that private repo in subsequent run commands: ```yml @@ -79,15 +91,7 @@ steps: run: app ``` -If you want to control how your command is passed to docker-compose, you can use the command parameter on the plugin directly: - -```yml -steps: - - plugins: - - docker-compose#v3.11.1: - run: app - command: ["custom", "command", "values"] -``` +Note, you will need to add the configuration to all steps in which you use this plugin. ## Artifacts @@ -166,11 +170,11 @@ steps: - MY_CUSTOM_ARG=panda ``` -Note that the values in the list must be a KEY=VALUE pair. +Note that the values in the list must be a `KEY=VALUE` pair. ## Pre-building the image -To speed up run steps that use the same service/image (such as steps that run in parallel), you can add a pre-build step to your pipeline: +If you have multiple steps that use the same service/image (such as steps that run in parallel), you can use this plugin in a specific `build` step to your pipeline. That will set specific metadata in the pipeline for this plugin to use in `run` steps afterwards: ```yml steps: @@ -190,7 +194,7 @@ steps: run: app ``` -All `run` steps for the service `app` will automatically pull and use the pre-built image. +All `run` steps for the service `app` will automatically pull and use the pre-built image. Without this, each `Test %n` job would build its own instead. ## Building multiple images @@ -226,19 +230,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v3.11.1: - push: app -``` - -If you need to authenticate to the repository to push (e.g. when pushing to Docker Hub), use the Docker Login plugin: - -```yml -steps: - - label: ":docker: Push" - plugins: - - docker-login#v2.0.1: - username: xyz - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: push: app ``` @@ -248,9 +240,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-login#v2.0.1: - username: xyz - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: push: - first-service - second-service @@ -262,9 +252,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-login#v2.0.1: - username: xyz - - docker-compose#v3.11.1: + - docker-compose#v3.11.2: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -350,7 +338,7 @@ are another (with a default name). The first successfully downloaded image in ea ### `build` -The name of a service to build and store, allowing following pipeline steps to run faster as they won't need to build the image. The step’s `command` will be ignored and does not need to be specified. +The name of a service to build and store, allowing following pipeline steps to run faster as they won't need to build the image. The step's `command` will be ignored and does not need to be specified. Either a single service or multiple services can be provided as an array. @@ -360,7 +348,7 @@ The name of the service the command should be run within. If the docker-compose ### `push` -A list of services to push in the format `service:image:tag`. If an image has been pre-built with the build step, that image will be re-tagged, otherwise docker-compose's built in push operation will be used. +A list of services to push in the format `service:image:tag`. If an image has been pre-built with the build step, that image will be re-tagged, otherwise docker-compose's built-in push operation will be used. ### `pull` (optional, run only) From f47d49e89ba3ad44d46dcdcf620c7b397ee64d10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sun, 25 Sep 2022 21:16:26 -0300 Subject: [PATCH 11/14] Updated version in preparation for next release --- README.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 15d914ea..88996047 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: run: app ``` @@ -28,7 +28,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: run: app config: docker-compose.tests.yml env: @@ -41,7 +41,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: run: app config: - docker-compose.yml @@ -56,7 +56,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: run: app ``` @@ -65,7 +65,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v3.11.2: + - docker-compose#v3.11.3: run: app command: ["custom", "command", "values"] ``` @@ -79,7 +79,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -87,7 +87,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: run: app ``` @@ -122,7 +122,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: run: app volumes: - "./dist:/app/dist" @@ -143,7 +143,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: run: app env: - BUILDKITE_BUILD_NUMBER @@ -163,7 +163,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -180,7 +180,7 @@ If you have multiple steps that use the same service/image (such as steps that r steps: - label: ":docker: Build" plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: build: app image-repository: index.docker.io/myorg/myrepo @@ -190,7 +190,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: run: app ``` @@ -206,7 +206,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: build: - app - tests @@ -218,7 +218,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: run: tests ``` @@ -230,7 +230,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v3.11.2: + - docker-compose#v3.11.3: push: app ``` @@ -240,7 +240,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-compose#v3.11.2: + - docker-compose#v3.11.3: push: - first-service - second-service @@ -252,7 +252,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-compose#v3.11.2: + - docker-compose#v3.11.3: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -266,14 +266,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v3.11.1: + - docker-compose#v3.11.3: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest From 9622b0f54adc37deab57d861a9ba5561730af35a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sun, 25 Sep 2022 23:32:23 -0300 Subject: [PATCH 12/14] Added tests for multiple-commands --- tests/multiple-commands.bats | 126 ++++++++++++++++++++++++++++++++--- 1 file changed, 116 insertions(+), 10 deletions(-) diff --git a/tests/multiple-commands.bats b/tests/multiple-commands.bats index 9224cd09..39aa0736 100644 --- a/tests/multiple-commands.bats +++ b/tests/multiple-commands.bats @@ -15,12 +15,12 @@ export BUILDKITE_JOB_ID=12 export BUILDKITE_PIPELINE_SLUG=test -@test "Build and run in a single step" { +@test "Build and run" { export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + + # necessary for build export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas - export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false - export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false stub docker-compose \ "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" \ @@ -28,24 +28,130 @@ export BUILDKITE_PIPELINE_SLUG=test "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran dependencies" \ "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml run --name buildkite12_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" + + # these commands simulate metadata for a specific value by using an intermediate-file + stub buildkite-agent \ + "meta-data set docker-compose-plugin-built-image-tag-myservice \* : echo \$4 > /tmp/build-run-metadata" \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : test -f /tmp/build-run-metadata" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : cat /tmp/build-run-metadata" + + run $PWD/hooks/command + + assert_success + assert_output --partial "Building services myservice" + assert_output --partial "Pushing built images to my.repository/llamas" + assert_output --partial "Found a pre-built image for myservice" + assert_output --partial "Starting dependencies" + assert_output --partial "ran myservice" + + unstub docker-compose + unstub buildkite-agent +} + +@test "Build and push" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH=myservice + + # necessary for build + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas + + stub docker-compose \ + "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" \ + "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml push myservice : echo build-pushed myservice" \ + "-f docker-compose.yml -p buildkite12 config : echo ''" \ + "-f docker-compose.yml -p buildkite12 push myservice : echo push-pushed myservice" + # these commands simulate metadata for a specific value by using an intermediate-file stub buildkite-agent \ - "meta-data set docker-compose-plugin-built-image-tag-myservice my.repository/llamas:test-myservice-build-1 : echo set meta-data" \ - "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ - "meta-data get docker-compose-plugin-built-image-tag-myservice : echo got meta-data" + "meta-data set docker-compose-plugin-built-image-tag-myservice \* : echo \$4 > /tmp/build-push-metadata" \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : test -f /tmp/build-push-metadata" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : cat /tmp/build-push-metadata" + + stub docker \ + "pull my.repository/llamas:test-myservice-build-1 : echo pulled pre-built image" \ + "tag my.repository/llamas:test-myservice-build-1 buildkite12_myservice : echo re-tagged pre-built image" run $PWD/hooks/command + assert_success + + assert_output --partial "Building services myservice" + assert_output --partial "Pushing built images to my.repository/llamas" + assert_output --partial "Pulling pre-built service myservice" + assert_output --partial "Tagging pre-built service myservice" + assert_output --partial "Pushing images for myservice" + unstub docker-compose unstub buildkite-agent +} + +@test "Run and push without pre-built image" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH=myservice + + stub docker-compose \ + "-f docker-compose.yml -p buildkite12 build --pull myservice : echo built myservice" \ + "-f docker-compose.yml -p buildkite12 up -d --scale myservice=0 myservice : echo ran dependencies" \ + "-f docker-compose.yml -p buildkite12 run --name buildkite12_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" \ + "-f docker-compose.yml -p buildkite12 config : echo ''" \ + "-f docker-compose.yml -p buildkite12 build myservice : echo built-2 myservice" \ + "-f docker-compose.yml -p buildkite12 push myservice : echo pushed myservice" + + # these make sure that the image is not pre-built + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1" + + run $PWD/hooks/command assert_success - assert_output --partial "built myservice" - assert_output --partial "pushed myservice" - assert_output --partial "pulled myservice" - assert_output --partial "ran dependencies" + + assert_output --partial "Building Docker Compose Service: myservice" + assert_output --partial "No pre-built image found from a previous " + assert_output --partial "Starting dependencies" assert_output --partial "ran myservice" + assert_output --partial "Building myservice" + assert_output --partial "Pushing images for myservice" + + unstub docker-compose + unstub buildkite-agent } +@test "Run and push with pre-built image" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PUSH=myservice + + stub docker-compose \ + "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo ran dependencies" \ + "-f docker-compose.yml -p buildkite12 -f docker-compose.buildkite-1-override.yml run --name buildkite12_myservice_build_1 --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice" \ + "-f docker-compose.yml -p buildkite12 config : echo ''" \ + "-f docker-compose.yml -p buildkite12 push myservice : echo pushed myservice" + + # these make sure that the image is not pre-built + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myservice-tag" \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myservice-tag" + + stub docker \ + "pull myservice-tag : echo pulled pre-built image" \ + "tag myservice-tag buildkite12_myservice : echo re-tagged pre-built image" + + run $PWD/hooks/command + + assert_success + + refute_output --partial "Building services myservice" + assert_output --partial "Found a pre-built image for myservice" + assert_output --partial "Pulling services myservice" + assert_output --partial "Starting dependencies" + assert_output --partial "Pulling pre-built service myservice" + assert_output --partial "Pushing images for myservice" + + unstub docker-compose + unstub buildkite-agent +} From e0357ad0c3b7f7dc5a8664613374bf8157e6eba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sun, 25 Sep 2022 23:32:38 -0300 Subject: [PATCH 13/14] Reworked command documentation a bit --- README.md | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 88996047..571356df 100644 --- a/README.md +++ b/README.md @@ -336,20 +336,56 @@ are another (with a default name). The first successfully downloaded image in ea ## Configuration -### `build` +### Main Commands + +You will need to specify at least one of the following to use this extension. + +#### `build` The name of a service to build and store, allowing following pipeline steps to run faster as they won't need to build the image. The step's `command` will be ignored and does not need to be specified. Either a single service or multiple services can be provided as an array. -### `run` +#### `run` The name of the service the command should be run within. If the docker-compose command would usually be `docker-compose run app test.sh` then the value would be `app`. -### `push` +#### `push` A list of services to push in the format `service:image:tag`. If an image has been pre-built with the build step, that image will be re-tagged, otherwise docker-compose's built-in push operation will be used. +#### Known issues + +##### Run & Push + +A basic pipeline similar to the following: + +```yaml +steps: + - label: ":docker: Run & Push" + plugins: + - docker-compose#v3.12.0: + run: myservice + push: myservice +``` + +Will cause the image to be built twice (once before running and once before pushing) unless there was a previous `build` step that set the appropriate metadata. + +##### Run & Push + +A basic pipeline similar to the following: + +```yaml +steps: + - label: ":docker: Build & Push" + plugins: + - docker-compose#v3.12.0: + build: myservice + push: myservice +``` + +Will cause the image to be pushed twice (once by the build step and another by the push step) + ### `pull` (optional, run only) Pull down multiple pre-built images. By default only the service that is being run will be pulled down, but this allows multiple images to be specified to handle prebuilt dependent images. From ae189c59f13002848dfeb94c75aca1c8a70a0fe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sun, 25 Sep 2022 23:36:36 -0300 Subject: [PATCH 14/14] Updated Readme version once more --- README.md | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 571356df..fb830350 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using steps: - command: test.sh plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: app ``` @@ -28,7 +28,7 @@ through if you need: steps: - command: test.sh plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: app config: docker-compose.tests.yml env: @@ -41,7 +41,7 @@ or multiple config files: steps: - command: test.sh plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: app config: - docker-compose.yml @@ -56,7 +56,7 @@ env: steps: - command: test.sh plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: app ``` @@ -65,7 +65,7 @@ If you want to control how your command is passed to docker-compose, you can use ```yml steps: - plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: app command: ["custom", "command", "values"] ``` @@ -79,7 +79,7 @@ steps: - plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: build: app image-repository: index.docker.io/myorg/myrepo - wait @@ -87,7 +87,7 @@ steps: plugins: - docker-login#v2.0.1: username: xyz - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: app ``` @@ -104,7 +104,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: app ``` @@ -122,7 +122,7 @@ steps: - command: generate-dist.sh artifact_paths: "dist/*" plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: app volumes: - "./dist:/app/dist" @@ -143,7 +143,7 @@ this plugin offers a `environment` block of its own: steps: - command: generate-dist.sh plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: app env: - BUILDKITE_BUILD_NUMBER @@ -163,7 +163,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th steps: - command: generate-dist.sh plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: build: app image-repository: index.docker.io/myorg/myrepo args: @@ -180,7 +180,7 @@ If you have multiple steps that use the same service/image (such as steps that r steps: - label: ":docker: Build" plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: build: app image-repository: index.docker.io/myorg/myrepo @@ -190,7 +190,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: app ``` @@ -206,7 +206,7 @@ steps: agents: queue: docker-builder plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: build: - app - tests @@ -218,7 +218,7 @@ steps: command: test.sh parallelism: 25 plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: run: tests ``` @@ -230,7 +230,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu steps: - label: ":docker: Push" plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: push: app ``` @@ -240,7 +240,7 @@ To push multiple images, you can use a list: steps: - label: ":docker: Push" plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: push: - first-service - second-service @@ -252,7 +252,7 @@ If you want to push to a specific location (that's not defined as the `image` in steps: - label: ":docker: Push" plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -266,14 +266,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w steps: - label: ":docker: Build an image" plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: app:index.docker.io/myorg/myrepo/myapp:latest - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v3.11.3: + - docker-compose#v3.12.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:latest @@ -287,7 +287,7 @@ This plugin allows for the value of `cache-from` to be a string or a list. If it steps: - label: ":docker Build an image" plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.12.0: build: app image-repository: index.docker.io/myorg/myrepo cache-from: @@ -296,7 +296,7 @@ steps: - wait - label: ":docker: Push to final repository" plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.12.0: push: - app:index.docker.io/myorg/myrepo/myapp - app:index.docker.io/myorg/myrepo/myapp:my-branch @@ -310,7 +310,7 @@ Adding a grouping tag to the end of a cache-from list item allows this plugin to steps: - label: ":docker: Build Intermediate Image" plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.12.0: build: myservice_intermediate # docker-compose.yml is the same as myservice but has `target: intermediate` image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo/myservice_intermediate @@ -320,7 +320,7 @@ steps: - wait - label: ":docker: Build Final Image" plugins: - - docker-compose#v3.9.0: + - docker-compose#v3.12.0: build: myservice image-name: buildkite-build-${BUILDKITE_BUILD_NUMBER} image-repository: index.docker.io/myorg/myrepo