Merge pull request #82 from dreyks/cache-from

basic cache_from support

Dockerfile

@@ -7,7 +7,7 @@ RUN mkdir -p /usr/local/lib/bats/bats-mock \
     && printf 'source "%s"\n' "/usr/local/lib/bats/bats-mock/stub.bash" >> /usr/local/lib/bats/load.bash \
     && rm -rf /tmp/bats-mock.tgz
 
-RUN apk --no-cache add ncurses
+RUN apk --no-cache add ncurses bc
 
 WORKDIR /app
 ENTRYPOINT ["/usr/local/bin/bats"]

hooks/commands/build.sh

@@ -2,8 +2,23 @@
 set -ueo pipefail
 
 image_repository="$(plugin_read_config IMAGE_REPOSITORY)"
+pull_retries="$(plugin_read_config PULL_RETRIES "0")"
 override_file="docker-compose.buildkite-${BUILDKITE_BUILD_NUMBER}-override.yml"
 build_images=()
+declare -A cache_from
+
+for line in $(plugin_read_list CACHE_FROM) ; do
+  IFS=':' read -a tokens <<< "$line"
+  service_name=${tokens[0]}
+  service_image=$(IFS=':'; echo "${tokens[*]:1}")
+
+  echo "~~~ :docker: Pulling cache image for $service_name"
+  if retry "$pull_retries" plugin_prompt_and_run docker pull "$service_image" ; then
+    cache_from[$service_name]=$service_image
+  else
+    echo "!!! :docker: Pull failed. $service_image will not be used as a cache for $service_name"
+  fi
+done
 
 for service_name in $(plugin_read_list BUILD) ; do
   image_name=$(build_image_name "${service_name}")
@@ -13,6 +28,12 @@ for service_name in $(plugin_read_list BUILD) ; do
   fi
 
   build_images+=("$service_name" "$image_name")
+
+  if in_array $service_name ${!cache_from[@]} ; then
+    build_images+=("${cache_from[$service_name]}")
+  else
+    build_images+=("")
+  fi
 done
 
 if [[ ${#build_images[@]} -gt 0 ]] ; then
@@ -31,6 +52,6 @@ if [[ -n "$image_repository" ]]; then
 
   while [[ ${#build_images[@]} -gt 0 ]] ; do
     plugin_set_metadata "built-image-tag-${build_images[0]}" "${build_images[1]}"
-    build_images=("${build_images[@]:2}")
+    build_images=("${build_images[@]:3}")
   done
 fi

hooks/commands/run.sh

@@ -26,7 +26,7 @@ test -f "$override_file" && rm "$override_file"
 
 if prebuilt_image=$(get_prebuilt_image "$service_name") ; then
   echo "~~~ :docker: Found a pre-built image for $service_name"
-  build_image_override_file "${service_name}" "${prebuilt_image}" | tee "$override_file"
+  build_image_override_file "${service_name}" "${prebuilt_image}" "" | tee "$override_file"
 
   echo "~~~ :docker: Pulling pre-built services $service_name"
   retry "$pull_retries" run_docker_compose -f "$override_file" pull "$service_name"

lib/shared.bash

@@ -102,14 +102,30 @@ function build_image_override_file_with_version() {
     exit 1
   fi
 
+  cache_from_not_available=($(bc <<< "$version < 3.2"))
+
   printf "version: '%s'\n" "$version"
   printf "services:\n"
 
   shift
   while test ${#} -gt 0 ; do
     printf "  %s:\n" "$1"
     printf "    image: %s\n" "$2"
-    shift 2
+
+    if [[ -n "$3" ]] ; then
+      if [[ $cache_from_not_available -gt 0 ]] ; then
+        echo "The 'cache_from' option can only be used with Compose file versions 3.2 and above."
+        echo "For more information on Docker Compose configuration file versions, see:"
+        echo "https://docs.docker.com/compose/compose-file/compose-versioning/#versioning"
+        exit 1
+      fi
+
+      printf "    build:\n"
+      printf "      cache_from:\n"
+      printf "        - %s\n" "$3"
+    fi
+
+    shift 3
   done
 }
 

tests/build.bats

@@ -86,3 +86,78 @@ load '../lib/shared'
   assert_failure
   assert_output --partial "Compose file versions 2.0 and above"
 }
+
+@test "Build with a cache-from image" {
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml"
+  export BUILDKITE_JOB_ID=1111
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:latest
+  export BUILDKITE_PIPELINE_SLUG=test
+  export BUILDKITE_BUILD_NUMBER=1
+
+  stub docker \
+    "pull my.repository/myservice_cache:latest : echo pulled cache image"
+
+  stub docker-compose \
+    "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld"
+
+  run $PWD/hooks/command
+
+  assert_success
+  assert_output --partial "pulled cache image"
+  assert_output --partial "- my.repository/myservice_cache:latest"
+  assert_output --partial "built helloworld"
+  unstub docker
+  unstub docker-compose
+}
+
+@test "Build with a cache-from image when pulling of the cache-from image failed" {
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml"
+  export BUILDKITE_JOB_ID=1111
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:latest
+  export BUILDKITE_PIPELINE_SLUG=test
+  export BUILDKITE_BUILD_NUMBER=1
+
+  stub docker \
+    "pull my.repository/myservice_cache:latest : exit 1"
+
+  stub docker-compose \
+    "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld"
+
+  run $PWD/hooks/command
+
+  assert_success
+  assert_output --partial "my.repository/myservice_cache:latest will not be used as a cache for helloworld"
+  refute_output --partial "- my.repository/myservice_cache:latest"
+  assert_output --partial "built helloworld"
+  unstub docker
+  unstub docker-compose
+}
+
+@test "Build with a cache-from image retry on failing pull" {
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml"
+  export BUILDKITE_JOB_ID=1111
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:latest
+  export BUILDKITE_PIPELINE_SLUG=test
+  export BUILDKITE_BUILD_NUMBER=1
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PULL_RETRIES=3
+
+  stub docker \
+    "pull my.repository/myservice_cache:latest : exit 1" \
+    "pull my.repository/myservice_cache:latest : exit 1" \
+    "pull my.repository/myservice_cache:latest : echo pulled cache image"
+
+  stub docker-compose \
+    "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld"
+
+  run $PWD/hooks/command
+
+  assert_success
+  assert_output --partial "pulled cache image"
+  assert_output --partial "- my.repository/myservice_cache:latest"
+  assert_output --partial "built helloworld"
+  unstub docker
+  unstub docker-compose
+}

tests/composefiles/docker-compose.v3.2.yml

@@ -0,0 +1,29 @@
+version: "3.2"
+
+services:
+  helloworld:
+    environment:
+      - BLAH=${VARIABLE:-default}
+    build: .
+
+  helloworldimage:
+    image: myhelloworld
+    build: .
+
+  alpinewithenv:
+    image: alpine:3.3
+    environment:
+      - LLAMAS=${MISSING_VARIABLE:-always}
+    volumes:
+      - ../../tests/scripts:/scripts:ro
+    command: /scripts/test_env.sh
+
+  alpinewithfailinglink:
+    image: alpine:3.3
+    command: echo hello from alpine
+    links:
+      - fail
+
+  fail:
+    image: alpine:3.3
+    command: sh -c "echo failing on purpose, expect exit 1; exit 1"

tests/image-override-file.bats

@@ -21,18 +21,42 @@ services:
 EOF
 )
 
+myservice_override_file3=$(cat <<-EOF
+version: '3.2'
+services:
+  myservice:
+    image: newimage:1.0.0
+    build:
+      cache_from:
+        - my.repository/myservice:latest
+EOF
+)
+
 @test "Build an docker-compose override file" {
-  run build_image_override_file_with_version "2.1" "myservice" "newimage:1.0.0"
+  run build_image_override_file_with_version "2.1" "myservice" "newimage:1.0.0" ""
 
   assert_success
   assert_output "$myservice_override_file1"
 }
 
 @test "Build an docker-compose override file with multiple entries" {
   run build_image_override_file_with_version "2.1" \
-    "myservice1" "newimage1:1.0.0" \
-    "myservice2" "newimage2:1.0.0"
+    "myservice1" "newimage1:1.0.0" "" \
+    "myservice2" "newimage2:1.0.0" ""
 
   assert_success
   assert_output "$myservice_override_file2"
 }
+
+@test "Build a docker-compose file with cache-from" {
+  run build_image_override_file_with_version "3.2" "myservice" "newimage:1.0.0" "my.repository/myservice:latest"
+
+  assert_success
+  assert_output "$myservice_override_file3"
+}
+
+@test "Build a docker-compose file with cache-from and compose-file version < 3.2" {
+  run build_image_override_file_with_version "3" "myservice" "newimage:1.0.0" "my.repository/myservice:latest"
+
+  assert_failure
+}