Minimal file digest for deps cache

For deps cache, builds hash based on a subset of files, mainly those lines that start with '#' (skipping whitespace).

This means deps cache only really misses when the dependency structure might possibly change.

Bug: b/348229942
Test: Unit tests updated, manual tests in chrome
Change-Id: I08b4643cfd51e2cd311e4189cb3f30f3acc86e0b
GitOrigin-RevId: c8a2273a10a9ebea8bd8138d93b79b1a0bc77c1d

internal/pkg/features/features.go

@@ -36,6 +36,10 @@ type Config struct {
 	// clangscandeps
 	ExperimentalGomaDepsCache bool
 
+	// ExperimentalGomaDepsCacheSize is the maximum number of elements to
+	// be stored in the cache.
+	ExperimentalGomaDepsCacheSize int
+
 	// ExperimentalSysrootDoNotUpload disables upload of the files/directories
 	// under the directory specified by the --sysroot flag.
 	ExperimentalSysrootDoNotUpload bool
@@ -66,4 +70,5 @@ func init() {
 	flag.BoolVar(&GetConfig().ExperimentalGomaDepsCache, "experimental_goma_deps_cache", false, "Use go deps cache with goma instead of goma's deps cache")
 	flag.BoolVar(&GetConfig().ExperimentalExitOnStuckActions, "experimental_exit_on_stuck_actions", false, "Stops reproxy with exit_code=1 if the command didn't finish within 2*reclient_timeout")
 	flag.BoolVar(&GetConfig().EnableCredentialCache, "enable_creds_cache", true, "If false, disables the credentials cache even if used auth mechanism supports it")
+	flag.IntVar(&GetConfig().ExperimentalGomaDepsCacheSize, "experimental_goma_deps_cache_size", 300000, "Maximum number of entries to hold in the experimental deps cache.")
 }

internal/pkg/inputprocessor/action/cppcompile/preprocessor_test.go

@@ -172,13 +172,14 @@ func TestComputeSpecWithDepsCache(t *testing.T) {
 	c := &Preprocessor{
 		CPPDepScanner:    s,
 		BasePreprocessor: &inputprocessor.BasePreprocessor{Ctx: ctx, FileMetadataCache: fmc},
-		DepsCache:        depscache.New(filemetadata.NewSingleFlightCache()),
+		DepsCache:        depscache.New(),
 	}
 
 	existingFiles := []string{
 		filepath.Clean("bin/clang++"),
 		filepath.Clean("src/test.cpp"),
 		filepath.Clean("include/foo/a"),
+		filepath.Clean("include/foo.h"),
 		filepath.Clean("out/dummy"),
 	}
 	er, cleanup := execroot.Setup(t, existingFiles)
@@ -200,6 +201,7 @@ func TestComputeSpecWithDepsCache(t *testing.T) {
 	want := &command.InputSpec{
 		Inputs: []string{
 			filepath.Clean("src/test.cpp"),
+			filepath.Clean("include/foo.h"),
 			filepath.Clean("bin/clang++"),
 		},
 		VirtualInputs: []*command.VirtualInput{
@@ -214,7 +216,7 @@ func TestComputeSpecWithDepsCache(t *testing.T) {
 	c = &Preprocessor{
 		CPPDepScanner:    s,
 		BasePreprocessor: &inputprocessor.BasePreprocessor{Ctx: ctx, FileMetadataCache: fmc},
-		DepsCache:        depscache.New(filemetadata.NewSingleFlightCache()),
+		DepsCache:        depscache.New(),
 	}
 	c.DepsCache.LoadFromDir(er)
 	got, err = inputprocessor.Compute(c, opts)
@@ -244,7 +246,7 @@ func TestComputeSpecWithDepsCache_ResourceDirChanged(t *testing.T) {
 			FileMetadataCache: fmc,
 			Executor:          &stubExecutor{outStr: "/first/resource/dir"},
 		},
-		DepsCache: depscache.New(filemetadata.NewSingleFlightCache()),
+		DepsCache: depscache.New(),
 	}
 
 	existingFiles := []string{
@@ -292,7 +294,7 @@ func TestComputeSpecWithDepsCache_ResourceDirChanged(t *testing.T) {
 			FileMetadataCache: fmc,
 			Executor:          &stubExecutor{outStr: "/second/resource/dir"},
 		},
-		DepsCache: depscache.New(filemetadata.NewSingleFlightCache()),
+		DepsCache: depscache.New(),
 	}
 	c.DepsCache.LoadFromDir(er)
 	got, err = inputprocessor.Compute(c, opts)
@@ -602,13 +604,14 @@ func TestComputeSpecEventTimes(t *testing.T) {
 	c := &Preprocessor{
 		CPPDepScanner:    s,
 		BasePreprocessor: &inputprocessor.BasePreprocessor{Ctx: ctx, FileMetadataCache: fmc},
-		DepsCache:        depscache.New(filemetadata.NewSingleFlightCache()),
+		DepsCache:        depscache.New(),
 	}
 
 	existingFiles := []string{
 		filepath.Clean("bin/clang++"),
 		filepath.Clean("src/test.cpp"),
 		filepath.Clean("include/foo/a"),
+		filepath.Clean("include/foo.h"),
 		filepath.Clean("out/dummy"),
 	}
 	er, cleanup := execroot.Setup(t, existingFiles)
@@ -641,7 +644,7 @@ func TestComputeSpecEventTimes(t *testing.T) {
 	c = &Preprocessor{
 		CPPDepScanner:    s,
 		BasePreprocessor: &inputprocessor.BasePreprocessor{Ctx: ctx, FileMetadataCache: fmc},
-		DepsCache:        depscache.New(filemetadata.NewSingleFlightCache()),
+		DepsCache:        depscache.New(),
 	}
 	c.DepsCache.LoadFromDir(er)
 	wg.Add(1)

internal/pkg/inputprocessor/depscache/BUILD.bazel

@@ -3,17 +3,19 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "depscache",
     srcs = [
-        "cppfilecache.go",
         "depscache.go",
+        "minimalfilecache.go",
     ],
     importpath = "github.com/bazelbuild/reclient/internal/pkg/inputprocessor/depscache",
     visibility = ["//:__subpackages__"],
     deps = [
         "//api/proxy",
         "//internal/pkg/event",
+        "//internal/pkg/features",
         "//internal/pkg/logger",
         "//internal/pkg/version",
-        "@com_github_bazelbuild_remote_apis_sdks//go/pkg/filemetadata",
+        "@com_github_bazelbuild_remote_apis_sdks//go/pkg/cache",
+        "@com_github_bazelbuild_remote_apis_sdks//go/pkg/digest",
         "@com_github_golang_glog//:glog",
         "@org_golang_google_protobuf//proto",
         "@org_golang_google_protobuf//types/known/timestamppb",

internal/pkg/inputprocessor/depscache/depscache.go

@@ -25,9 +25,9 @@ import (
 
 	ppb "github.com/bazelbuild/reclient/api/proxy"
 	"github.com/bazelbuild/reclient/internal/pkg/event"
+	"github.com/bazelbuild/reclient/internal/pkg/features"
 	"github.com/bazelbuild/reclient/internal/pkg/logger"
 	"github.com/bazelbuild/reclient/internal/pkg/version"
-	"github.com/bazelbuild/remote-apis-sdks/go/pkg/filemetadata"
 	log "github.com/golang/glog"
 	"google.golang.org/protobuf/proto"
 
@@ -52,7 +52,7 @@ type Cache struct {
 	Logger     *logger.Logger
 	depsCache  map[Key][]*ppb.FileInfo
 	depsMu     sync.RWMutex
-	filesCache cppFileCache
+	filesCache minimalFileCache
 	// last use time of a key
 	lutByKey map[Key]time.Time
 	lutMu    sync.Mutex
@@ -62,13 +62,12 @@ type Cache struct {
 }
 
 // New creates a new empty deps cache.
-func New(fmc filemetadata.Cache) *Cache {
+func New() *Cache {
 	return &Cache{
-		MaxEntries: 100000,
+		MaxEntries: features.GetConfig().ExperimentalGomaDepsCacheSize,
 		depsCache:  make(map[Key][]*ppb.FileInfo),
 		lutByKey:   make(map[Key]time.Time),
-		filesCache: cppFileCache{
-			fmc:   fmc,
+		filesCache: minimalFileCache{
 			files: make(map[string]fileInfo),
 		},
 	}