Add the remote_require_cached flag

When set to true, this flag causes Bazel to abort the build whenever it
encounters an action that is not cached.

This is very useful when trying to troubleshoot action non-determinism
issues because it allows running a build and having it fail as soon as
there is a problem without tainting what already exists in the cache.

My workflow is to essentially do:

1. bazel build ...
2. bazel clean
3. bazel build --remote_require_cached ...

which makes step 3 fail on the first non-deterministic action. Then I can
address that problem and re-run step 3 to encounter the next issue.

src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java

@@ -236,6 +236,19 @@ public SpawnResult exec(Spawn spawn, SpawnExecutionContext context)
       return execLocallyAndUploadOrFail(action, spawn, context, uploadLocalResults, e);
     }
 
+    if (remoteOptions.remoteRequireCached) {
+      return new SpawnResult.Builder()
+          .setStatus(SpawnResult.Status.EXECUTION_DENIED)
+          .setExitCode(1)
+          .setFailureMessage("Action must be cached due to --remote_require_cached but it is not")
+          .setFailureDetail(
+              FailureDetail.newBuilder()
+                  .setSpawn(FailureDetails.Spawn.newBuilder().setCode(FailureDetails.Spawn.Code.EXECUTION_DENIED))
+                  .build())
+          .setRunnerName("remote")
+          .build();
+    }
+
     AtomicBoolean useCachedResult = new AtomicBoolean(acceptCachedResult);
     AtomicBoolean forceUploadInput = new AtomicBoolean(false);
     try {

src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java

@@ -229,6 +229,18 @@ public final class RemoteOptions extends CommonRemoteOptions {
       help = "Whether to accept remotely cached action results.")
   public boolean remoteAcceptCached;
 
+  @Option(
+      name = "remote_require_cached",
+      defaultValue = "false",
+      documentationCategory = OptionDocumentationCategory.REMOTE,
+      effectTags = {OptionEffectTag.UNKNOWN},
+      help =
+          "If set to true, enforce that all actions that can run remotely are cached, or else "
+              + "fail the build. This is useful to troubleshoot non-determinism issues as it "
+              + "allows checking whether actions that should be cached are actually cached "
+              + "without spuriously injecting new results into the cache.")
+  public boolean remoteRequireCached;
+
   @Option(
       name = "remote_local_fallback",
       defaultValue = "false",

src/test/shell/bazel/remote/remote_execution_test.sh

@@ -1263,6 +1263,36 @@ EOF
   expect_not_log "1 local"
 }
 
+function test_require_cached() {
+  mkdir -p a
+  cat > a/BUILD <<'EOF'
+genrule(
+  name = "foo",
+  srcs = ["foo.in"],
+  outs = ["foo.out"],
+  cmd = "cp \"$<\" \"$@\"",
+)
+EOF
+
+  echo "input 1" >a/foo.in
+  bazel build \
+    --remote_executor=grpc://localhost:${worker_port} \
+    //a:foo >& $TEST_log || fail "Failed to build //a:foo"
+
+  expect_log "1 remote"
+
+  echo "input 2" >a/foo.in
+  if bazel build \
+    --remote_executor=grpc://localhost:${worker_port} \
+    --remote_require_cached \
+    //a:foo >& $TEST_log; then
+    fail "Build of //a:foo succeeded but it should have failed"
+  fi
+
+  expect_log "Action must be cached due to --remote_require_cached but it is not"
+  expect_not_log "remote cache hit"
+}
+
 function test_nobuild_runfile_links() {
   mkdir data && echo "hello" > data/hello && echo "world" > data/world
     cat > WORKSPACE <<EOF