Merge bitcoin/bitcoin#31545: ci: optionally use local docker build cache

e87429a ci: optionally use local docker build cache (0xb10c)

Pull request description:

  By setting `DANGER_DOCKER_BUILD_CACHE_HOST_DIR`, the task-specific docker images built during the CI run can be cached. This allows, for example, ephemeral CI runners to reuse the docker images (or layers of it) from earlier runs, by persisting the image cache before the ephemeral CI runner is shut down. The cache keyed by `CONTAINER_NAME`.

  As `--cache-to` doesn't remove old cache files, the existing cache is removed after a successful `docker build` and the newly cached image is moved to it's location to avoid the cache from growing indefinitely with old, unused layers.

  When `--cache-from` doesn't find the directory, the cached version is a cache-miss, or the cache can't be imported for whatever other reason, it warns and `docker build` continues by building the docker image.

  This feature is opt-in. The documentation for the docker build cache of `type=local` can be found on https://docs.docker.com/build/cache/backends/local/

  This replaces bitcoin/bitcoin#31377 - some of the discussion there might provide more context.

ACKs for top commit:
  maflcko:
    I haven't tested this, and it looks harmless and is easy to revert, if needed. So lgtm ACK e87429a
  achow101:
    ACK e87429a
  TheCharlatan:
    tACK e87429a
  willcl-ark:
    ACK e87429a

Tree-SHA512: 0887c395dee2e2020394933246d4c1bfb6dde7165219cbe93eccfe01379e05c75dce8920b6edd7df07364c703fcee7be4fba8fa45fd0e0e89da9e24759f67a71

ci/test/02_run_container.sh

@@ -25,6 +25,29 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
   fi
   echo "Creating $CI_IMAGE_NAME_TAG container to run in"
 
+  DOCKER_BUILD_CACHE_ARG=""
+  DOCKER_BUILD_CACHE_TEMPDIR=""
+  DOCKER_BUILD_CACHE_OLD_DIR=""
+  DOCKER_BUILD_CACHE_NEW_DIR=""
+  # If set, use an `docker build` cache directory on the CI host
+  # to cache docker image layers for the CI container image.
+  # This cache can be multiple GB in size. Prefixed with DANGER
+  # as setting it removes (old cache) files from the host.
+  if [ "$DANGER_DOCKER_BUILD_CACHE_HOST_DIR" ]; then
+    # Directory where the current cache for this run could be. If not existing
+    # or empty, "docker build" will warn, but treat it as cache-miss and continue.
+    DOCKER_BUILD_CACHE_OLD_DIR="${DANGER_DOCKER_BUILD_CACHE_HOST_DIR}/${CONTAINER_NAME}"
+    # Temporary directory for a newly created cache. We can't write the new
+    # cache into OLD_DIR directly, as old cache layers would not be removed.
+    # The NEW_DIR contents are moved to OLD_DIR after OLD_DIR has been cleared.
+    # This happens after `docker build`. If a task fails or is aborted, the
+    # DOCKER_BUILD_CACHE_TEMPDIR might be retained on the host. If the host isn't
+    # ephemeral, it has to take care of cleaning old TEMPDIR's up.
+    DOCKER_BUILD_CACHE_TEMPDIR="$(mktemp --directory ci-docker-build-cache-XXXXXXXXXX)"
+    DOCKER_BUILD_CACHE_NEW_DIR="${DOCKER_BUILD_CACHE_TEMPDIR}/${CONTAINER_NAME}"
+    DOCKER_BUILD_CACHE_ARG="--cache-from type=local,src=${DOCKER_BUILD_CACHE_OLD_DIR} --cache-to type=local,dest=${DOCKER_BUILD_CACHE_NEW_DIR},mode=max"
+  fi
+
   # shellcheck disable=SC2086
   DOCKER_BUILDKIT=1 docker build \
       --file "${BASE_READ_ONLY_DIR}/ci/test_imagefile" \
@@ -34,8 +57,18 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
       --platform="${CI_IMAGE_PLATFORM}" \
       --label="${CI_IMAGE_LABEL}" \
       --tag="${CONTAINER_NAME}" \
+      $DOCKER_BUILD_CACHE_ARG \
       "${BASE_READ_ONLY_DIR}"
 
+  if [ "$DANGER_DOCKER_BUILD_CACHE_HOST_DIR" ]; then
+    if [ -e "${DOCKER_BUILD_CACHE_NEW_DIR}/index.json" ]; then
+      echo "Removing the existing docker build cache in ${DOCKER_BUILD_CACHE_OLD_DIR}"
+      rm -rf "${DOCKER_BUILD_CACHE_OLD_DIR}"
+      echo "Moving the contents of ${DOCKER_BUILD_CACHE_NEW_DIR} to ${DOCKER_BUILD_CACHE_OLD_DIR}"
+      mv "${DOCKER_BUILD_CACHE_NEW_DIR}" "${DOCKER_BUILD_CACHE_OLD_DIR}"
+    fi
+  fi
+
   docker volume create "${CONTAINER_NAME}_ccache" || true
   docker volume create "${CONTAINER_NAME}_depends" || true
   docker volume create "${CONTAINER_NAME}_depends_sources" || true