From bbf2fa37a6792de01ccdbf6568535859b9719131 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 24 Sep 2022 03:49:03 -0300 Subject: [PATCH 1/2] Added option to expand vars in volume configurations --- README.md | 8 ++++++++ lib/run.bash | 11 ++++++++++- plugin.yml | 3 +++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8765d1a9..bc146ff0 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,8 @@ steps: - "./dist:/app/dist" ``` +If you want to use environment variables in the `volumes` element, you will need to activate the (unsafe) option `expand-volume-vars`. + ## Environment By default, docker-compose makes whatever environment variables it gets available for @@ -462,6 +464,12 @@ A list of volumes to mount into the container. If a matching volume exists in th Additionally, volumes may be specified via the agent environment variable `BUILDKITE_DOCKER_DEFAULT_VOLUMES`, a `;` (semicolon) delimited list of mounts in the `-v` syntax. (Ex. `buildkite:/buildkite;./app:/app`). +### `expand-volume-vars` (optional, boolean, run only, unsafe) + +When set to true, it will activate interpolation of variables in the elements of the `volumes` configuration array. When turned off (the default), attempting to use variables will fail as the literal `$VARIABLE_NAME` string will be passed to the `-v` option. + +:warning: **Important:** this is considered an unsafe option as the most compatible way to achieve this is to run the strings through `eval` which could lead to arbitrary code execution or information leaking if you don't have complete control of the pipeline + ### `graceful-shutdown` (optional, run only) Gracefully shuts down all containers via 'docker-compose stop`. diff --git a/lib/run.bash b/lib/run.bash index 3c88be92..91697fdf 100644 --- a/lib/run.bash +++ b/lib/run.bash @@ -76,7 +76,14 @@ check_linked_containers_and_save_logs() { # # "./foo:/foo" => "/buildkite/builds/.../foo:/foo" expand_relative_volume_path() { - local path="$1" + local path + + if [[ "$(plugin_read_config EXPAND_VOLUME_VARS 'false')" == "true" ]]; then + path=$(eval echo "$1") + else + path="$1" + fi + local pwd="$PWD" # docker-compose's -v expects native paths on windows, so convert back. @@ -86,6 +93,8 @@ expand_relative_volume_path() { pwd="$(cygpath -w "$PWD")" fi + + echo "${path/.\//$pwd/}" } diff --git a/plugin.yml b/plugin.yml index b0a1d41f..b6478e0a 100644 --- a/plugin.yml +++ b/plugin.yml @@ -46,6 +46,8 @@ configuration: volumes: type: [ string, array ] minimum: 1 + expand-volume-vars: + type: boolean command: type: array skip-checkout: @@ -95,6 +97,7 @@ configuration: push-retries: [ push ] cache-from: [ build ] volumes: [ run ] + expand-volume-vars: [ volumes ] leave-volumes: [ run ] use-aliases: [ run ] dependencies: [ run ] From db2f7d6e02114ecdbc5ea89ff2302f29ccfe8813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20A=2E=20Bellone?= Date: Sat, 24 Sep 2022 03:49:14 -0300 Subject: [PATCH 2/2] Add tests to the new volume option --- tests/run.bats | 104 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/tests/run.bats b/tests/run.bats index 7b445310..916705f9 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -610,6 +610,110 @@ export BUILDKITE_JOB_ID=1111 unstub buildkite-agent } +@test "Run with volumes with variables" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_0="\$SUPER_VARIABLE:/mnt" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_1="/:\$OTHER_VARIABLE" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_2="\$RELATIVE_VARIABLE:/srv" + + export SUPER_VARIABLE='/test/path' + export OTHER_VARIABLE='/path/tested' + export RELATIVE_VARIABLE='./path' + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 -v \\\$SUPER_VARIABLE:/mnt -v /:\\\$OTHER_VARIABLE -v \\\$RELATIVE_VARIABLE:/srv --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with volumes" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "ran myservice with volumes" + unstub docker-compose + unstub buildkite-agent +} + + +@test "Run with volumes with variables but option turned off" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_EXPAND_VOLUME_VARS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_0="\$SUPER_VARIABLE:/mnt" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_1="/:\$OTHER_VARIABLE" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_2="\$RELATIVE_VARIABLE:/srv" + + + export SUPER_VARIABLE='/test/path' + export OTHER_VARIABLE='/path/tested' + export RELATIVE_VARIABLE='./path' + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 -v \\\$SUPER_VARIABLE:/mnt -v /:\\\$OTHER_VARIABLE -v \\\$RELATIVE_VARIABLE:/srv --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with volumes" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "ran myservice with volumes" + unstub docker-compose + unstub buildkite-agent +} + +@test "Run with volumes with variables and option turned on" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_COMMAND=pwd + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=true + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_EXPAND_VOLUME_VARS=true + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_0="\$SUPER_VARIABLE:/mnt" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_1="/:\$OTHER_VARIABLE" + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_2="\$RELATIVE_VARIABLE:/srv" + + + export SUPER_VARIABLE='/test/path' + export OTHER_VARIABLE='/path/tested' + export RELATIVE_VARIABLE='./path' + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 -v /test/path:/mnt -v /:/path/tested -v $PWD/path:/srv --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with volumes" + + stub buildkite-agent \ + "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \ + "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage" + + run $PWD/hooks/command + + assert_success + assert_output --partial "ran myservice with volumes" + unstub docker-compose + unstub buildkite-agent +} + @test "Run with default volumes" { export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice