Multiple manifest support

README.md

@@ -13,7 +13,7 @@ steps:
   - label: ':nodejs: Install dependencies'
     command: npm ci
     plugins:
-      - cache#v1.4.0:
+      - cache#v1.5.0:
           manifest: package-lock.json
           path: node_modules
           restore: file
@@ -77,16 +77,16 @@ Assuming the underlying executables are available, the allowed values are:
 
 Force saving the cache even if it exists. Default: `false`.
 
-### `manifest` (string, required if using `file` caching level)
+### `manifest` (string or list of strings, required if using `file` caching level)
 
-A path to a file or folder that will be hashed to create file-level caches.
+One or more paths to files or folders that will be hashed to create and restore file-level caches. If multiple files or folders are specified its ordering does not matter.
 
 It will cause an unrecoverable error if either `save` or `restore` are set to `file` and this option is not specified.
 
 ## Caching levels
 
 This plugin uses the following hierarchical structure for caches to be valid (meaning usable), from the most specific to the more general:
-* `file`: only as long as a manifest file does not change (see the `manifest` option)
+* `file`: only as long as the contents of the files or folders of the `manifest` option do not change
 * `step`: valid only for the current step
 * `branch`: when the pipeline executes in the context of the current branch
 * `pipeline`: all builds and steps of the pipeline
@@ -131,7 +131,7 @@ steps:
   - label: ':nodejs: Install dependencies'
     command: npm ci
     plugins:
-      - cache#v1.4.0:
+      - cache#v1.5.0:
           manifest: package-lock.json
           path: node_modules
           restore: pipeline
@@ -142,7 +142,7 @@ steps:
   - label: ':test_tube: Run tests'
     command: npm test # does not save cache, not necessary
     plugins:
-      - cache#v1.4.0:
+      - cache#v1.5.0:
           manifest: package-lock.json
           path: node_modules
           restore: file
@@ -151,7 +151,7 @@ steps:
     if: build.branch == "master"
     command: npm run deploy
     plugins:
-      - cache#v1.4.0:
+      - cache#v1.5.0:
           manifest: package-lock.json
           path: node_modules
           restore: file

lib/shared.bash

@@ -23,9 +23,12 @@ sha() {
   echo "$shasum"
 }
 
-# Hashes files and directories recursively
+# Hashes multiple files and directories recursively
 hash_files() {
-  find "$@" -type f -print0 \
+  ( for FILE in "$@"; do
+      find "$FILE" -type f -print0
+    done
+  ) | sort -z \
     | xargs -0 "$(sha)" \
     | cut -d\  -f1 \
     | sort \
@@ -39,7 +42,8 @@ build_key() {
   local COMPRESSION="${3:-}"
 
   if [ "${LEVEL}" = 'file' ]; then
-    BASE="$(hash_files "$(plugin_read_config MANIFEST)")"
+    plugin_read_list_into_result MANIFEST
+    BASE="$(hash_files "${result[@]}")"
   elif [ "${LEVEL}" = 'step' ]; then
     BASE="${BUILDKITE_PIPELINE_SLUG}${BUILDKITE_LABEL}"
   elif [ "${LEVEL}" = 'branch' ]; then

plugin.yml

@@ -24,7 +24,11 @@ configuration:
     force:
       type: boolean
     manifest:
-      type: string
+      oneOf:
+        - type: string
+        - type: array
+          items:
+            type: string
     path:
       type: string
     restore:

tests/shared-multiple-manifests.bats

@@ -0,0 +1,282 @@
+#!/usr/bin/env bats
+
+# To debug stubs, uncomment these lines:
+# export HASH_STUB_DEBUG=/dev/tty
+
+setup() {
+  load "${BATS_PLUGIN_PATH}/load.bash"
+
+  source "${PWD}/lib/shared.bash"
+}
+
+teardown() {
+  rm -rf test.file other.file third.file test.folder other.folder FOLDER FOLDER2
+}
+
+@test 'file build_key when manifest is list with single file works' {
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_0=test.file
+  touch test.file
+
+  run build_key file FOLDER
+
+  assert_success
+
+  EMPTY_FILE="${output}"
+
+  # changing what will be cached, changes the output
+  run build_key file FOLDER2
+
+  assert_success
+  refute_output "${EMPTY_FILE}"
+
+  # changing the manifest changes the output
+  echo 'new_value' > test.file
+
+  run build_key file FOLDER
+
+  assert_success
+  refute_output "${EMPTY_FILE}"
+  MODIFIED_FILE="${output}"
+
+  # adding things outside the manifest does not change
+  mkdir FOLDER
+  touch FOLDER/test.file
+  touch other.file
+
+  run build_key file FOLDER
+
+  assert_success
+  refute_output "${EMPTY_FILE}"
+  assert_output "${MODIFIED_FILE}"
+
+  rm test.file
+  rm -rf FOLDER
+  rm other.file
+}
+
+@test 'file build_key with multiple manifest files does not depend on order' {
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_0=test.file
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_1=other.file
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_2=third.file
+
+  echo '1' > test.file
+  echo '2' > other.file
+  echo '3' > third.file
+
+  run build_key file FOLDER
+  EXPECTED="${output}"
+
+  # change ordering
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_0=test.file
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_1=third.file
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_2=other.file
+
+  run build_key file FOLDER
+  assert_output "${EXPECTED}"
+
+  # change ordering
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_0=other.file
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_1=test.file
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_2=third.file
+
+  run build_key file FOLDER
+  assert_output "${EXPECTED}"
+
+  # change ordering
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_0=other.file
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_1=third.file
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_2=test.file
+
+  run build_key file FOLDER
+  assert_output "${EXPECTED}"
+
+  # change ordering
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_0=third.file
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_1=test.file
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_2=other.file
+
+  run build_key file FOLDER
+  assert_output "${EXPECTED}"
+
+  # change ordering
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_0=third.file
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_1=other.file
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_2=test.file
+
+  run build_key file FOLDER
+  assert_output "${EXPECTED}"
+
+  rm test.file
+  rm other.file
+  rm third.file
+}
+
+@test 'file build_key with multiple manifest files works' {
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_0=test.file
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_1=other.file
+
+  touch test.file
+  touch other.file
+
+  run build_key file FOLDER
+
+  assert_success
+
+  EMPTY_FILE="${output}"
+
+  # changing what will be cached, changes the output
+  run build_key file FOLDER2
+
+  assert_success
+  refute_output "${EMPTY_FILE}"
+
+  # changing the manifest in one file changes the output
+  echo 'new_value' > test.file
+
+  run build_key file FOLDER
+
+  assert_success
+  refute_output "${EMPTY_FILE}"
+  MODIFIED_FILE_1="${output}"
+
+  # adding things outside the manifest does not change
+  mkdir FOLDER
+  touch FOLDER/test.file
+  touch other.file
+
+  run build_key file FOLDER
+
+  assert_success
+  refute_output "${EMPTY_FILE}"
+  assert_output "${MODIFIED_FILE_1}"
+
+  # changing the manifest in the other file changes the output
+  rm test.file && touch test.file # restoring the original test.file
+  echo 'other_value' > other.file
+
+  run build_key file FOLDER
+
+  assert_success
+  refute_output "${EMPTY_FILE}"
+  refute_output "${MODIFIED_FILE_1}"
+
+  rm test.file
+  rm -rf FOLDER
+  rm other.file
+}
+
+@test 'file build_key with multiple manifest files & folders works' {
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_0=test.folder
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_1=test.file
+
+  mkdir test.folder
+  touch test.file
+
+  run build_key file FOLDER
+
+  assert_success
+  EMPTY_FOLDER="${output}"
+
+  # changing what will be cached, changes the output
+  run build_key file FOLDER2
+
+  assert_success
+  refute_output "${EMPTY_FOLDER}"
+
+  # changing the ordering does not change the output
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_0=test.file
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_1=test.folder
+
+  run build_key file FOLDER
+
+  assert_success
+  assert_output "${EMPTY_FOLDER}"
+
+  # changing the contents of the folder changes the output
+  echo 'new_value' > test.folder/test.file
+
+  run build_key file FOLDER
+
+  assert_success
+  refute_output "${EMPTY_FOLDER}"
+  MODIFIED_FOLDER="${output}"
+
+  # adding things outside the manifest does not change
+  mkdir FOLDER
+  touch FOLDER/test.file
+  touch other.file
+
+  run build_key file FOLDER
+
+  assert_success
+  refute_output "${EMPTY_FOLDER}"
+  assert_output "${MODIFIED_FOLDER}"
+
+  rm -rf test.folder
+  rm test.file
+  rm other.file
+  rm -rf FOLDER
+}
+
+
+@test 'file build_key with multiple manifest folders works' {
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_0=test.folder
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_1=other.folder
+
+  mkdir test.folder
+  mkdir other.folder
+  echo '1' > other.folder/test.file
+
+  run build_key file FOLDER
+
+  assert_success
+  EMPTY_FOLDER="${output}"
+
+  # changing what will be cached, changes the output
+  run build_key file FOLDER2
+
+  assert_success
+  refute_output "${EMPTY_FOLDER}"
+
+  # changing the ordering does not change the output
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_0=other.folder
+  export BUILDKITE_PLUGIN_CACHE_MANIFEST_1=test.folder
+
+  run build_key file FOLDER
+
+  assert_success
+  assert_output "${EMPTY_FOLDER}"
+
+  # changing the contents of the folder changes the output
+  echo 'new_value' > test.folder/test.file
+
+  run build_key file FOLDER
+
+  assert_success
+  refute_output "${EMPTY_FOLDER}"
+  MODIFIED_FOLDER="${output}"
+
+  # adding things outside the manifest does not change
+  mkdir FOLDER
+  touch FOLDER/test.file
+  touch other.file
+
+  run build_key file FOLDER
+
+  assert_success
+  refute_output "${EMPTY_FOLDER}"
+  assert_output "${MODIFIED_FOLDER}"
+
+  # removing something from the folder changes the output
+  rm other.folder/test.file
+  run build_key file FOLDER
+
+  assert_success
+  refute_output "${EMPTY_FOLDER}"
+  refute_output "${MODIFIED_FOLDER}"
+
+  rm -rf test.folder
+  rm -rf other.folder
+  rm other.file
+  rm -rf FOLDER
+}