From 9fc7857b8c9943701623bb89550329646b7676be Mon Sep 17 00:00:00 2001 From: Roman Usherenko Date: Wed, 15 Nov 2017 14:28:58 +0200 Subject: [PATCH 1/4] basic cache-from support --- hooks/commands/build.sh | 10 ++++++++++ lib/shared.bash | 13 +++++++++++++ tests/build.bats | 22 ++++++++++++++++++++++ tests/image-override-file.bats | 20 ++++++++++++++++++++ 4 files changed, 65 insertions(+) diff --git a/hooks/commands/build.sh b/hooks/commands/build.sh index f6ee71fc..1c942b9f 100755 --- a/hooks/commands/build.sh +++ b/hooks/commands/build.sh @@ -22,6 +22,16 @@ fi services=( $(plugin_read_list BUILD) ) + +for line in $(plugin_read_list CACHE_FROM) ; do + IFS=':' read -a tokens <<< "$line" + service_name=${tokens[0]} + service_image=$(IFS=':'; echo "${tokens[*]:1}") + + echo "+++ :docker: Pulling cache image for $service_name" + plugin_prompt_and_run docker pull "$service_image" +done + echo "+++ :docker: Building services ${services[*]}" run_docker_compose -f "$override_file" build --pull "${services[@]}" diff --git a/lib/shared.bash b/lib/shared.bash index ed6210c1..7a15b9c2 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -94,6 +94,12 @@ function build_image_override_file() { # docker-compose version and set of service and image pairs function build_image_override_file_with_version() { local version="$1" + declare -A cache_from + + for line in $(plugin_read_list CACHE_FROM) ; do + IFS=':' read -a tokens <<< "$line" + cache_from[${tokens[0]}]=$(IFS=':'; echo "${tokens[*]:1}") + done if [[ -z "$version" ]]; then echo "The 'build' option can only be used with Compose file versions 2.0 and above." @@ -102,6 +108,8 @@ function build_image_override_file_with_version() { exit 1 fi + # TODO: cache_from requires version: 3.2 + printf "version: '%s'\n" "$version" printf "services:\n" @@ -109,6 +117,11 @@ function build_image_override_file_with_version() { while test ${#} -gt 0 ; do printf " %s:\n" "$1" printf " image: %s\n" "$2" + if in_array $1 ${!cache_from[@]} ; then + printf " build:\n" + printf " cache_from:\n" + printf " - %s\n" ${cache_from[$1]} + fi shift 2 done } diff --git a/tests/build.bats b/tests/build.bats index f955d12b..a6e06b67 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -86,3 +86,25 @@ load '../lib/shared' assert_failure assert_output --partial "Compose file versions 2.0 and above" } + +@test "Build with a cache-from image" { + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=myservice:my.repository/myservice:latest + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "pull my.repository/myservice:latest : echo pulled cache image" + + stub docker-compose \ + "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" + + run $PWD/hooks/command + + assert_success + assert_output --partial "pulled cache image" + assert_output --partial "built myservice" + unstub docker + unstub docker-compose +} diff --git a/tests/image-override-file.bats b/tests/image-override-file.bats index 2dae472c..8f9dc04c 100644 --- a/tests/image-override-file.bats +++ b/tests/image-override-file.bats @@ -21,6 +21,17 @@ services: EOF ) +myservice_override_file3=$(cat <<-EOF +version: '3.2' +services: + myservice: + image: newimage:1.0.0 + build: + cache_from: + - my.repository/myservice:latest +EOF +) + @test "Build an docker-compose override file" { run build_image_override_file_with_version "2.1" "myservice" "newimage:1.0.0" @@ -36,3 +47,12 @@ EOF assert_success assert_output "$myservice_override_file2" } + +@test "Build a docker-compose file with cache-from" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=myservice:my.repository/myservice:latest + + run build_image_override_file_with_version "3.2" "myservice" "newimage:1.0.0" + + assert_success + assert_output "$myservice_override_file3" +} From e454d2bef1498fc5020a9d4201b9986fd468386d Mon Sep 17 00:00:00 2001 From: Roman Usherenko Date: Thu, 16 Nov 2017 17:37:38 +0200 Subject: [PATCH 2/4] fail when trying to use cache_from with version < 3.2 --- Dockerfile | 2 +- lib/shared.bash | 8 ++++++ tests/build.bats | 7 +++--- tests/composefiles/docker-compose.v3.2.yml | 29 ++++++++++++++++++++++ tests/image-override-file.bats | 8 ++++++ 5 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 tests/composefiles/docker-compose.v3.2.yml diff --git a/Dockerfile b/Dockerfile index 800147ad..9b8e6cc4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ RUN mkdir -p /usr/local/lib/bats/bats-mock \ && printf 'source "%s"\n' "/usr/local/lib/bats/bats-mock/stub.bash" >> /usr/local/lib/bats/load.bash \ && rm -rf /tmp/bats-mock.tgz -RUN apk --no-cache add ncurses +RUN apk --no-cache add ncurses bc WORKDIR /app ENTRYPOINT ["/usr/local/bin/bats"] diff --git a/lib/shared.bash b/lib/shared.bash index 7a15b9c2..b203f174 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -100,6 +100,7 @@ function build_image_override_file_with_version() { IFS=':' read -a tokens <<< "$line" cache_from[${tokens[0]}]=$(IFS=':'; echo "${tokens[*]:1}") done + local using_cache_from=${cache_from[@]+"${#cache_from[@]}"} if [[ -z "$version" ]]; then echo "The 'build' option can only be used with Compose file versions 2.0 and above." @@ -108,6 +109,13 @@ function build_image_override_file_with_version() { exit 1 fi + if [[ -n "$using_cache_from" && $(bc <<< "$version < 3.2") -gt 0 ]] ; then + echo "The 'cache_from' option can only be used with Compose file versions 3.2 and above." + echo "For more information on Docker Compose configuration file versions, see:" + echo "https://docs.docker.com/compose/compose-file/compose-versioning/#versioning" + exit 1 + fi + # TODO: cache_from requires version: 3.2 printf "version: '%s'\n" "$version" diff --git a/tests/build.bats b/tests/build.bats index a6e06b67..392f7dd7 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -88,8 +88,9 @@ load '../lib/shared' } @test "Build with a cache-from image" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" export BUILDKITE_JOB_ID=1111 - export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD=myservice + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=myservice:my.repository/myservice:latest export BUILDKITE_PIPELINE_SLUG=test export BUILDKITE_BUILD_NUMBER=1 @@ -98,13 +99,13 @@ load '../lib/shared' "pull my.repository/myservice:latest : echo pulled cache image" stub docker-compose \ - "-f docker-compose.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull myservice : echo built myservice" + "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" run $PWD/hooks/command assert_success assert_output --partial "pulled cache image" - assert_output --partial "built myservice" + assert_output --partial "built helloworld" unstub docker unstub docker-compose } diff --git a/tests/composefiles/docker-compose.v3.2.yml b/tests/composefiles/docker-compose.v3.2.yml new file mode 100644 index 00000000..5cd3b32d --- /dev/null +++ b/tests/composefiles/docker-compose.v3.2.yml @@ -0,0 +1,29 @@ +version: "3.2" + +services: + helloworld: + environment: + - BLAH=${VARIABLE:-default} + build: . + + helloworldimage: + image: myhelloworld + build: . + + alpinewithenv: + image: alpine:3.3 + environment: + - LLAMAS=${MISSING_VARIABLE:-always} + volumes: + - ../../tests/scripts:/scripts:ro + command: /scripts/test_env.sh + + alpinewithfailinglink: + image: alpine:3.3 + command: echo hello from alpine + links: + - fail + + fail: + image: alpine:3.3 + command: sh -c "echo failing on purpose, expect exit 1; exit 1" diff --git a/tests/image-override-file.bats b/tests/image-override-file.bats index 8f9dc04c..3d2d32b6 100644 --- a/tests/image-override-file.bats +++ b/tests/image-override-file.bats @@ -56,3 +56,11 @@ EOF assert_success assert_output "$myservice_override_file3" } + +@test "Build a docker-compose file with cache-from and compose-file version < 3.2" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=myservice:my.repository/myservice:latest + + run build_image_override_file_with_version "3" "myservice" "newimage:1.0.0" + + assert_failure +} From fb8b88e507f352c7720604f7d2aa2583295d154b Mon Sep 17 00:00:00 2001 From: Roman Usherenko Date: Fri, 17 Nov 2017 11:27:58 +0200 Subject: [PATCH 3/4] remove code duplication when reading cache_from --- hooks/commands/build.sh | 30 +++++++++++++++++++----------- hooks/commands/run.sh | 2 +- lib/shared.bash | 31 +++++++++++++------------------ tests/image-override-file.bats | 14 +++++--------- 4 files changed, 38 insertions(+), 39 deletions(-) diff --git a/hooks/commands/build.sh b/hooks/commands/build.sh index 1c942b9f..7af9470a 100755 --- a/hooks/commands/build.sh +++ b/hooks/commands/build.sh @@ -4,6 +4,18 @@ set -ueo pipefail image_repository="$(plugin_read_config IMAGE_REPOSITORY)" override_file="docker-compose.buildkite-${BUILDKITE_BUILD_NUMBER}-override.yml" build_images=() +declare -A cache_from + +for line in $(plugin_read_list CACHE_FROM) ; do + IFS=':' read -a tokens <<< "$line" + service_name=${tokens[0]} + service_image=$(IFS=':'; echo "${tokens[*]:1}") + cache_from[$service_name]=$(IFS=':'; echo "$service_image") + + echo "+++ :docker: Pulling cache image for $service_name" + plugin_prompt_and_run docker pull "$service_image" +done +# using_cache_from=${cache_from[@]+"${#cache_from[@]}"} for service_name in $(plugin_read_list BUILD) ; do image_name=$(build_image_name "${service_name}") @@ -13,6 +25,12 @@ for service_name in $(plugin_read_list BUILD) ; do fi build_images+=("$service_name" "$image_name") + + if in_array $service_name ${!cache_from[@]} ; then + build_images+=("${cache_from[$1]}") + else + build_images+=("") + fi done if [[ ${#build_images[@]} -gt 0 ]] ; then @@ -22,16 +40,6 @@ fi services=( $(plugin_read_list BUILD) ) - -for line in $(plugin_read_list CACHE_FROM) ; do - IFS=':' read -a tokens <<< "$line" - service_name=${tokens[0]} - service_image=$(IFS=':'; echo "${tokens[*]:1}") - - echo "+++ :docker: Pulling cache image for $service_name" - plugin_prompt_and_run docker pull "$service_image" -done - echo "+++ :docker: Building services ${services[*]}" run_docker_compose -f "$override_file" build --pull "${services[@]}" @@ -41,6 +49,6 @@ if [[ -n "$image_repository" ]]; then while [[ ${#build_images[@]} -gt 0 ]] ; do plugin_set_metadata "built-image-tag-${build_images[0]}" "${build_images[1]}" - build_images=("${build_images[@]:2}") + build_images=("${build_images[@]:3}") done fi diff --git a/hooks/commands/run.sh b/hooks/commands/run.sh index 0d4de960..4093f33a 100755 --- a/hooks/commands/run.sh +++ b/hooks/commands/run.sh @@ -26,7 +26,7 @@ test -f "$override_file" && rm "$override_file" if prebuilt_image=$(get_prebuilt_image "$service_name") ; then echo "~~~ :docker: Found a pre-built image for $service_name" - build_image_override_file "${service_name}" "${prebuilt_image}" | tee "$override_file" + build_image_override_file "${service_name}" "${prebuilt_image}" "" | tee "$override_file" echo "~~~ :docker: Pulling pre-built services $service_name" retry "$pull_retries" run_docker_compose -f "$override_file" pull "$service_name" diff --git a/lib/shared.bash b/lib/shared.bash index b203f174..8b980770 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -94,13 +94,6 @@ function build_image_override_file() { # docker-compose version and set of service and image pairs function build_image_override_file_with_version() { local version="$1" - declare -A cache_from - - for line in $(plugin_read_list CACHE_FROM) ; do - IFS=':' read -a tokens <<< "$line" - cache_from[${tokens[0]}]=$(IFS=':'; echo "${tokens[*]:1}") - done - local using_cache_from=${cache_from[@]+"${#cache_from[@]}"} if [[ -z "$version" ]]; then echo "The 'build' option can only be used with Compose file versions 2.0 and above." @@ -109,14 +102,7 @@ function build_image_override_file_with_version() { exit 1 fi - if [[ -n "$using_cache_from" && $(bc <<< "$version < 3.2") -gt 0 ]] ; then - echo "The 'cache_from' option can only be used with Compose file versions 3.2 and above." - echo "For more information on Docker Compose configuration file versions, see:" - echo "https://docs.docker.com/compose/compose-file/compose-versioning/#versioning" - exit 1 - fi - - # TODO: cache_from requires version: 3.2 + cache_from_not_available=($(bc <<< "$version < 3.2")) printf "version: '%s'\n" "$version" printf "services:\n" @@ -125,12 +111,21 @@ function build_image_override_file_with_version() { while test ${#} -gt 0 ; do printf " %s:\n" "$1" printf " image: %s\n" "$2" - if in_array $1 ${!cache_from[@]} ; then + + if [[ -n "$3" ]] ; then + if [[ $cache_from_not_available -gt 0 ]] ; then + echo "The 'cache_from' option can only be used with Compose file versions 3.2 and above." + echo "For more information on Docker Compose configuration file versions, see:" + echo "https://docs.docker.com/compose/compose-file/compose-versioning/#versioning" + exit 1 + fi + printf " build:\n" printf " cache_from:\n" - printf " - %s\n" ${cache_from[$1]} + printf " - %s\n" "$3" fi - shift 2 + + shift 3 done } diff --git a/tests/image-override-file.bats b/tests/image-override-file.bats index 3d2d32b6..d06b6111 100644 --- a/tests/image-override-file.bats +++ b/tests/image-override-file.bats @@ -33,7 +33,7 @@ EOF ) @test "Build an docker-compose override file" { - run build_image_override_file_with_version "2.1" "myservice" "newimage:1.0.0" + run build_image_override_file_with_version "2.1" "myservice" "newimage:1.0.0" "" assert_success assert_output "$myservice_override_file1" @@ -41,26 +41,22 @@ EOF @test "Build an docker-compose override file with multiple entries" { run build_image_override_file_with_version "2.1" \ - "myservice1" "newimage1:1.0.0" \ - "myservice2" "newimage2:1.0.0" + "myservice1" "newimage1:1.0.0" "" \ + "myservice2" "newimage2:1.0.0" "" assert_success assert_output "$myservice_override_file2" } @test "Build a docker-compose file with cache-from" { - export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=myservice:my.repository/myservice:latest - - run build_image_override_file_with_version "3.2" "myservice" "newimage:1.0.0" + run build_image_override_file_with_version "3.2" "myservice" "newimage:1.0.0" "my.repository/myservice:latest" assert_success assert_output "$myservice_override_file3" } @test "Build a docker-compose file with cache-from and compose-file version < 3.2" { - export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=myservice:my.repository/myservice:latest - - run build_image_override_file_with_version "3" "myservice" "newimage:1.0.0" + run build_image_override_file_with_version "3" "myservice" "newimage:1.0.0" "my.repository/myservice:latest" assert_failure } From b681d47d7f33a0affdb0c4bd296114251330867c Mon Sep 17 00:00:00 2001 From: Roman Usherenko Date: Fri, 17 Nov 2017 13:19:22 +0200 Subject: [PATCH 4/4] do not set cache_from if the image failed to pull --- hooks/commands/build.sh | 13 ++++++---- tests/build.bats | 56 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 7 deletions(-) diff --git a/hooks/commands/build.sh b/hooks/commands/build.sh index 7af9470a..391f9dcb 100755 --- a/hooks/commands/build.sh +++ b/hooks/commands/build.sh @@ -2,6 +2,7 @@ set -ueo pipefail image_repository="$(plugin_read_config IMAGE_REPOSITORY)" +pull_retries="$(plugin_read_config PULL_RETRIES "0")" override_file="docker-compose.buildkite-${BUILDKITE_BUILD_NUMBER}-override.yml" build_images=() declare -A cache_from @@ -10,12 +11,14 @@ for line in $(plugin_read_list CACHE_FROM) ; do IFS=':' read -a tokens <<< "$line" service_name=${tokens[0]} service_image=$(IFS=':'; echo "${tokens[*]:1}") - cache_from[$service_name]=$(IFS=':'; echo "$service_image") - echo "+++ :docker: Pulling cache image for $service_name" - plugin_prompt_and_run docker pull "$service_image" + echo "~~~ :docker: Pulling cache image for $service_name" + if retry "$pull_retries" plugin_prompt_and_run docker pull "$service_image" ; then + cache_from[$service_name]=$service_image + else + echo "!!! :docker: Pull failed. $service_image will not be used as a cache for $service_name" + fi done -# using_cache_from=${cache_from[@]+"${#cache_from[@]}"} for service_name in $(plugin_read_list BUILD) ; do image_name=$(build_image_name "${service_name}") @@ -27,7 +30,7 @@ for service_name in $(plugin_read_list BUILD) ; do build_images+=("$service_name" "$image_name") if in_array $service_name ${!cache_from[@]} ; then - build_images+=("${cache_from[$1]}") + build_images+=("${cache_from[$service_name]}") else build_images+=("") fi diff --git a/tests/build.bats b/tests/build.bats index 392f7dd7..07e1f029 100644 --- a/tests/build.bats +++ b/tests/build.bats @@ -91,12 +91,12 @@ load '../lib/shared' export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" export BUILDKITE_JOB_ID=1111 export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld - export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=myservice:my.repository/myservice:latest + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:latest export BUILDKITE_PIPELINE_SLUG=test export BUILDKITE_BUILD_NUMBER=1 stub docker \ - "pull my.repository/myservice:latest : echo pulled cache image" + "pull my.repository/myservice_cache:latest : echo pulled cache image" stub docker-compose \ "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" @@ -105,6 +105,58 @@ load '../lib/shared' assert_success assert_output --partial "pulled cache image" + assert_output --partial "- my.repository/myservice_cache:latest" + assert_output --partial "built helloworld" + unstub docker + unstub docker-compose +} + +@test "Build with a cache-from image when pulling of the cache-from image failed" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:latest + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + + stub docker \ + "pull my.repository/myservice_cache:latest : exit 1" + + stub docker-compose \ + "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" + + run $PWD/hooks/command + + assert_success + assert_output --partial "my.repository/myservice_cache:latest will not be used as a cache for helloworld" + refute_output --partial "- my.repository/myservice_cache:latest" + assert_output --partial "built helloworld" + unstub docker + unstub docker-compose +} + +@test "Build with a cache-from image retry on failing pull" { + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml" + export BUILDKITE_JOB_ID=1111 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:latest + export BUILDKITE_PIPELINE_SLUG=test + export BUILDKITE_BUILD_NUMBER=1 + export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PULL_RETRIES=3 + + stub docker \ + "pull my.repository/myservice_cache:latest : exit 1" \ + "pull my.repository/myservice_cache:latest : exit 1" \ + "pull my.repository/myservice_cache:latest : echo pulled cache image" + + stub docker-compose \ + "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" + + run $PWD/hooks/command + + assert_success + assert_output --partial "pulled cache image" + assert_output --partial "- my.repository/myservice_cache:latest" assert_output --partial "built helloworld" unstub docker unstub docker-compose