Merge pull request #339 from buildkite-plugins/toote_volume_interpola…

…tion_issue-324

Volume variable interpolation

README.md

@@ -128,6 +128,8 @@ steps:
             - "./dist:/app/dist"
 ```
 
+If you want to use environment variables in the `volumes` element, you will need to activate the (unsafe) option `expand-volume-vars`.
+
 ## Environment
 
 By default, docker-compose makes whatever environment variables it gets available for
@@ -486,6 +488,12 @@ A list of volumes to mount into the container. If a matching volume exists in th
 
 Additionally, volumes may be specified via the agent environment variable `BUILDKITE_DOCKER_DEFAULT_VOLUMES`, a `;` (semicolon)  delimited list of mounts in the `-v` syntax. (Ex. `buildkite:/buildkite;./app:/app`).
 
+### `expand-volume-vars` (optional, boolean, run only, unsafe)
+
+When set to true, it will activate interpolation of variables in the elements of the `volumes` configuration array. When turned off (the default), attempting to use variables will fail as the literal `$VARIABLE_NAME` string will be passed to the `-v` option.
+
+:warning: **Important:** this is considered an unsafe option as the most compatible way to achieve this is to run the strings through `eval` which could lead to arbitrary code execution or information leaking if you don't have complete control of the pipeline
+
 ### `graceful-shutdown` (optional, run only)
 
 Gracefully shuts down all containers via 'docker-compose stop`.

lib/run.bash

@@ -76,7 +76,14 @@ check_linked_containers_and_save_logs() {
 #
 # "./foo:/foo" => "/buildkite/builds/.../foo:/foo"
 expand_relative_volume_path() {
-  local path="$1"
+  local path
+
+  if [[ "$(plugin_read_config EXPAND_VOLUME_VARS 'false')" == "true" ]]; then
+    path=$(eval echo "$1")
+  else
+    path="$1"
+  fi
+
   local pwd="$PWD"
 
   # docker-compose's -v expects native paths on windows, so convert back.
@@ -86,6 +93,8 @@ expand_relative_volume_path() {
     pwd="$(cygpath -w "$PWD")"
   fi
 
+
+
   echo "${path/.\//$pwd/}"
 }
 

plugin.yml

@@ -46,6 +46,8 @@ configuration:
     volumes:
       type: [ string, array ]
       minimum: 1
+    expand-volume-vars:
+      type: boolean
     command:
       type: array
     skip-checkout:
@@ -100,6 +102,7 @@ configuration:
     push-retries: [ push ]
     cache-from: [ build ]
     volumes: [ run ]
+    expand-volume-vars: [ volumes ]
     leave-volumes: [ run ]
     use-aliases: [ run ]
     dependencies: [ run ]

tests/run.bats

@@ -610,6 +610,110 @@ export BUILDKITE_JOB_ID=1111
   unstub buildkite-agent
 }
 
+@test "Run with volumes with variables" {
+  export BUILDKITE_JOB_ID=1111
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice
+  export BUILDKITE_PIPELINE_SLUG=test
+  export BUILDKITE_BUILD_NUMBER=1
+  export BUILDKITE_COMMAND=pwd
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_0="\$SUPER_VARIABLE:/mnt"
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_1="/:\$OTHER_VARIABLE"
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_2="\$RELATIVE_VARIABLE:/srv"
+
+  export SUPER_VARIABLE='/test/path'
+  export OTHER_VARIABLE='/path/tested'
+  export RELATIVE_VARIABLE='./path'
+
+  stub docker-compose \
+    "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \
+    "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \
+    "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 -v \\\$SUPER_VARIABLE:/mnt -v /:\\\$OTHER_VARIABLE -v \\\$RELATIVE_VARIABLE:/srv --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with volumes"
+
+  stub buildkite-agent \
+    "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \
+    "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage"
+
+  run $PWD/hooks/command
+
+  assert_success
+  assert_output --partial "ran myservice with volumes"
+  unstub docker-compose
+  unstub buildkite-agent
+}
+
+
+@test "Run with volumes with variables but option turned off" {
+  export BUILDKITE_JOB_ID=1111
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice
+  export BUILDKITE_PIPELINE_SLUG=test
+  export BUILDKITE_BUILD_NUMBER=1
+  export BUILDKITE_COMMAND=pwd
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_EXPAND_VOLUME_VARS=false
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_0="\$SUPER_VARIABLE:/mnt"
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_1="/:\$OTHER_VARIABLE"
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_2="\$RELATIVE_VARIABLE:/srv"
+
+
+  export SUPER_VARIABLE='/test/path'
+  export OTHER_VARIABLE='/path/tested'
+  export RELATIVE_VARIABLE='./path'
+
+  stub docker-compose \
+    "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \
+    "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \
+    "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 -v \\\$SUPER_VARIABLE:/mnt -v /:\\\$OTHER_VARIABLE -v \\\$RELATIVE_VARIABLE:/srv --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with volumes"
+
+  stub buildkite-agent \
+    "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \
+    "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage"
+
+  run $PWD/hooks/command
+
+  assert_success
+  assert_output --partial "ran myservice with volumes"
+  unstub docker-compose
+  unstub buildkite-agent
+}
+
+@test "Run with volumes with variables and option turned on" {
+  export BUILDKITE_JOB_ID=1111
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice
+  export BUILDKITE_PIPELINE_SLUG=test
+  export BUILDKITE_BUILD_NUMBER=1
+  export BUILDKITE_COMMAND=pwd
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=true
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_EXPAND_VOLUME_VARS=true
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_0="\$SUPER_VARIABLE:/mnt"
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_1="/:\$OTHER_VARIABLE"
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_VOLUMES_2="\$RELATIVE_VARIABLE:/srv"
+
+
+  export SUPER_VARIABLE='/test/path'
+  export OTHER_VARIABLE='/path/tested'
+  export RELATIVE_VARIABLE='./path'
+
+  stub docker-compose \
+    "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml pull myservice : echo pulled myservice" \
+    "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml up -d --scale myservice=0 myservice : echo started dependencies for myservice" \
+    "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml run --name buildkite1111_myservice_build_1 -v /test/path:/mnt -v /:/path/tested -v $PWD/path:/srv --rm myservice /bin/sh -e -c 'pwd' : echo ran myservice with volumes"
+
+  stub buildkite-agent \
+    "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 0" \
+    "meta-data get docker-compose-plugin-built-image-tag-myservice : echo myimage"
+
+  run $PWD/hooks/command
+
+  assert_success
+  assert_output --partial "ran myservice with volumes"
+  unstub docker-compose
+  unstub buildkite-agent
+}
+
 @test "Run with default volumes" {
   export BUILDKITE_JOB_ID=1111
   export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice