diff --git a/Dockerfile b/Dockerfile index 800147ad..9b8e6cc4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ RUN mkdir -p /usr/local/lib/bats/bats-mock \ && printf 'source "%s"\n' "/usr/local/lib/bats/bats-mock/stub.bash" >> /usr/local/lib/bats/load.bash \ && rm -rf /tmp/bats-mock.tgz -RUN apk --no-cache add ncurses +RUN apk --no-cache add ncurses bc WORKDIR /app ENTRYPOINT ["/usr/local/bin/bats"] diff --git a/hooks/commands/build.sh b/hooks/commands/build.sh index f6ee71fc..391f9dcb 100755 --- a/hooks/commands/build.sh +++ b/hooks/commands/build.sh @@ -2,8 +2,23 @@ set -ueo pipefail image_repository="$(plugin_read_config IMAGE_REPOSITORY)" +pull_retries="$(plugin_read_config PULL_RETRIES "0")" override_file="docker-compose.buildkite-${BUILDKITE_BUILD_NUMBER}-override.yml" build_images=() +declare -A cache_from + +for line in $(plugin_read_list CACHE_FROM) ; do + IFS=':' read -a tokens <<< "$line" + service_name=${tokens[0]} + service_image=$(IFS=':'; echo "${tokens[*]:1}") + + echo "~~~ :docker: Pulling cache image for $service_name" + if retry "$pull_retries" plugin_prompt_and_run docker pull "$service_image" ; then + cache_from[$service_name]=$service_image + else + echo "!!! :docker: Pull failed. $service_image will not be used as a cache for $service_name" + fi +done for service_name in $(plugin_read_list BUILD) ; do image_name=$(build_image_name "${service_name}") @@ -13,6 +28,12 @@ for service_name in $(plugin_read_list BUILD) ; do fi build_images+=("$service_name" "$image_name") + + if in_array $service_name ${!cache_from[@]} ; then + build_images+=("${cache_from[$service_name]}") + else + build_images+=("") + fi done if [[ ${#build_images[@]} -gt 0 ]] ; then @@ -31,6 +52,6 @@ if [[ -n "$image_repository" ]]; then while [[ ${#build_images[@]} -gt 0 ]] ; do plugin_set_metadata "built-image-tag-${build_images[0]}" "${build_images[1]}" - build_images=("${build_images[@]:2}") + build_images=("${build_images[@]:3}") done fi diff --git a/hooks/commands/run.sh b/hooks/commands/run.sh index 0d4de960..4093f33a 100755 --- a/hooks/commands/run.sh +++ b/hooks/commands/run.sh @@ -26,7 +26,7 @@ test -f "$override_file" && rm "$override_file" if prebuilt_image=$(get_prebuilt_image "$service_name") ; then echo "~~~ :docker: Found a pre-built image for $service_name" - build_image_override_file "${service_name}" "${prebuilt_image}" | tee "$override_file" + build_image_override_file "${service_name}" "${prebuilt_image}" "" | tee "$override_file" echo "~~~ :docker: Pulling pre-built services $service_name" retry "$pull_retries" run_docker_compose -f "$override_file" pull "$service_name" diff --git a/lib/shared.bash b/lib/shared.bash index ed6210c1..8b980770 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -102,6 +102,8 @@ function build_image_override_file_with_version() { exit 1 fi + cache_from_not_available=($(bc <<< "$version < 3.2")) + printf "version: '%s'\n" "$version" printf "services:\n" @@ -109,7 +111,21 @@ function build_image_override_file_with_version() { while test ${#} -gt 0 ; do printf " %s:\n" "$1" printf " image: %s\n" "$2" - shift 2 + + if [[ -n "$3" ]] ; then + if [[ $cache_from_not_available -gt 0 ]] ; then + echo "The 'cache_from' option can only be used with Compose file versions 3.2 and above." + echo "For more information on Docker Compose configuration file versions, see:" + echo "https://docs.docker.com/compose/compose-file/compose-versioning/#versioning" + exit 1 + fi + + printf " build:\n" + printf " cache_from:\n" + printf " - %s\n" "$3" + fi + + shift 3 done } diff --git a/tests/build.bats b/tests/build.bats index f955d12b..07e1f029 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -86,3 +86,78 @@ load '../lib/shared' assert_failure assert_output --partial "Compose file versions 2.0 and above" } + +@test "Build with a cache-from image" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:latest + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "pull my.repository/myservice_cache:latest : echo pulled cache image" + + stub docker-compose \ + "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" + + run $PWD/hooks/command + + assert_success + assert_output --partial "pulled cache image" + assert_output --partial "- my.repository/myservice_cache:latest" + assert_output --partial "built helloworld" + unstub docker + unstub docker-compose +} + +@test "Build with a cache-from image when pulling of the cache-from image failed" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:latest + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "pull my.repository/myservice_cache:latest : exit 1" + + stub docker-compose \ + "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" + + run $PWD/hooks/command + + assert_success + assert_output --partial "my.repository/myservice_cache:latest will not be used as a cache for helloworld" + refute_output --partial "- my.repository/myservice_cache:latest" + assert_output --partial "built helloworld" + unstub docker + unstub docker-compose +} + +@test "Build with a cache-from image retry on failing pull" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:latest + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PULL_RETRIES=3 + + stub docker \ + "pull my.repository/myservice_cache:latest : exit 1" \ + "pull my.repository/myservice_cache:latest : exit 1" \ + "pull my.repository/myservice_cache:latest : echo pulled cache image" + + stub docker-compose \ + "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" + + run $PWD/hooks/command + + assert_success + assert_output --partial "pulled cache image" + assert_output --partial "- my.repository/myservice_cache:latest" + assert_output --partial "built helloworld" + unstub docker + unstub docker-compose +} diff --git a/tests/composefiles/docker-compose.v3.2.yml b/tests/composefiles/docker-compose.v3.2.yml new file mode 100644 index 00000000..5cd3b32d --- /dev/null +++ b/tests/composefiles/docker-compose.v3.2.yml @@ -0,0 +1,29 @@ +version: "3.2" + +services: + helloworld: + environment: + - BLAH=${VARIABLE:-default} + build: . + + helloworldimage: + image: myhelloworld + build: . + + alpinewithenv: + image: alpine:3.3 + environment: + - LLAMAS=${MISSING_VARIABLE:-always} + volumes: + - ../../tests/scripts:/scripts:ro + command: /scripts/test_env.sh + + alpinewithfailinglink: + image: alpine:3.3 + command: echo hello from alpine + links: + - fail + + fail: + image: alpine:3.3 + command: sh -c "echo failing on purpose, expect exit 1; exit 1" diff --git a/tests/image-override-file.bats b/tests/image-override-file.bats index 2dae472c..d06b6111 100644 --- a/tests/image-override-file.bats +++ b/tests/image-override-file.bats @@ -21,8 +21,19 @@ services: EOF ) +myservice_override_file3=$(cat <<-EOF +version: '3.2' +services: + myservice: + image: newimage:1.0.0 + build: + cache_from: + - my.repository/myservice:latest +EOF +) + @test "Build an docker-compose override file" { - run build_image_override_file_with_version "2.1" "myservice" "newimage:1.0.0" + run build_image_override_file_with_version "2.1" "myservice" "newimage:1.0.0" "" assert_success assert_output "$myservice_override_file1" @@ -30,9 +41,22 @@ EOF @test "Build an docker-compose override file with multiple entries" { run build_image_override_file_with_version "2.1" \ - "myservice1" "newimage1:1.0.0" \ - "myservice2" "newimage2:1.0.0" + "myservice1" "newimage1:1.0.0" "" \ + "myservice2" "newimage2:1.0.0" "" assert_success assert_output "$myservice_override_file2" } + +@test "Build a docker-compose file with cache-from" { + run build_image_override_file_with_version "3.2" "myservice" "newimage:1.0.0" "my.repository/myservice:latest" + + assert_success + assert_output "$myservice_override_file3" +} + +@test "Build a docker-compose file with cache-from and compose-file version < 3.2" { + run build_image_override_file_with_version "3" "myservice" "newimage:1.0.0" "my.repository/myservice:latest" + + assert_failure +}