Skip to content

Commit

Permalink
Add execlog test with nested runfiles middleman
Browse files Browse the repository at this point in the history 
Verifies that runfiles middlemen artifacts in non-top-level nested sets are handled. Also makes the tests more realistic by ensuring that tools are always a subset of inputs.

Related to #23884

Closes #23890.

PiperOrigin-RevId: 683259163
Change-Id: Ic71377c069970038f20c0bd27bb7f719f3daef70
  • Loading branch information
@fmeum @copybara-github
fmeum authored and copybara-github committed Oct 7, 2024
1 parent cf90a71 commit 07668e6
Show file tree
Hide file tree
Showing 2 changed files with 73 additions and 24 deletions.
4 changes: 2 additions & 2 deletions src/test/java/com/google/devtools/build/lib/exec/CompactSpawnLogContextTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,14 +185,14 @@ public void testRunfilesTreeReusedForTool() throws Exception {

context.logSpawn(
firstSpawn,
createInputMetadataProvider(toolRunfilesMiddleman, runfilesTree, firstInput),
createInputMetadataProvider(runfilesTree, toolRunfilesMiddleman, firstInput),
createInputMap(runfilesTree, firstInput),
fs,
defaultTimeout(),
defaultSpawnResult());
context.logSpawn(
secondSpawn,
createInputMetadataProvider(toolRunfilesMiddleman, runfilesTree, secondInput),
createInputMetadataProvider(runfilesTree, toolRunfilesMiddleman, secondInput),
createInputMap(runfilesTree, secondInput),
fs,
defaultTimeout(),
Expand Down
93 changes: 71 additions & 22 deletions src/test/java/com/google/devtools/build/lib/exec/SpawnLogContextTestBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.Iterables;
import com.google.devtools.build.lib.actions.ActionEnvironment;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.Artifact;
Expand Down Expand Up @@ -430,7 +429,7 @@ public void testRunfilesFileInput(@TestParameter InputsMode inputsMode) throws E

context.logSpawn(
spawnBuilder.build(),
createInputMetadataProvider(runfilesMiddleman, runfilesTree, runfilesInput),
createInputMetadataProvider(runfilesTree, runfilesMiddleman, runfilesInput),
createInputMap(runfilesTree),
fs,
defaultTimeout(),
Expand All @@ -451,6 +450,60 @@ public void testRunfilesFileInput(@TestParameter InputsMode inputsMode) throws E
.build());
}

@Test
public void testRunfilesNestedMiddleman() throws Exception {
Artifact runfilesMiddleman = ActionsTestUtil.createArtifact(middlemanDir, "runfiles");
Artifact runfilesInput = ActionsTestUtil.createArtifact(rootDir, "data.txt");
writeFile(runfilesInput, "abc");
Artifact toolFile1 = ActionsTestUtil.createArtifact(rootDir, "tool1");
writeFile(toolFile1, "def");
Artifact toolFile2 = ActionsTestUtil.createArtifact(rootDir, "tool2");
writeFile(toolFile2, "ghi");

PathFragment runfilesRoot = outputDir.getExecPath().getRelative("foo.runfiles");
RunfilesTree runfilesTree = createRunfilesTree(runfilesRoot, runfilesInput);

NestedSet<ActionInput> tools =
NestedSetBuilder.<ActionInput>stableOrder()
.add(toolFile1)
.addTransitive(
NestedSetBuilder.<ActionInput>stableOrder()
.add(runfilesMiddleman)
.add(toolFile2)
.build())
.build();
Spawn spawn = defaultSpawnBuilder().withInputs(tools).withTools(tools).build();

SpawnLogContext context = createSpawnLogContext();

context.logSpawn(
spawn,
createInputMetadataProvider(
runfilesTree, runfilesMiddleman, runfilesInput, toolFile1, toolFile2),
createInputMap(runfilesTree, toolFile1, toolFile2),
fs,
defaultTimeout(),
defaultSpawnResult());

closeAndAssertLog(
context,
defaultSpawnExecBuilder()
.addInputs(
File.newBuilder()
.setPath(
PRODUCT_NAME
+ "-out/k8-fastbuild/bin/foo.runfiles/"
+ WORKSPACE_NAME
+ "/data.txt")
.setDigest(getDigest("abc"))
.setIsTool(true))
.addInputs(
File.newBuilder().setPath("tool1").setDigest(getDigest("def")).setIsTool(true))
.addInputs(
File.newBuilder().setPath("tool2").setDigest(getDigest("ghi")).setIsTool(true))
.build());
}

@Test
public void testRunfilesDirectoryInput(
@TestParameter DirContents dirContents, @TestParameter InputsMode inputsMode)
Expand All @@ -475,7 +528,7 @@ public void testRunfilesDirectoryInput(

context.logSpawn(
spawnBuilder.build(),
createInputMetadataProvider(runfilesMiddleman, runfilesTree, runfilesInput),
createInputMetadataProvider(runfilesTree, runfilesMiddleman, runfilesInput),
createInputMap(runfilesTree),
fs,
defaultTimeout(),
Expand Down Expand Up @@ -539,8 +592,8 @@ public void testRunfilesEmptyInput(@TestParameter InputsMode inputsMode) throws
context.logSpawn(
spawnBuilder.build(),
createInputMetadataProvider(
runfilesMiddleman,
runfilesTree,
runfilesMiddleman,
runfilesInput,
externalGenArtifact,
externalSourceArtifact),
Expand Down Expand Up @@ -685,8 +738,8 @@ public void testRunfilesMixedRoots(@TestParameter boolean legacyExternalRunfiles
context.logSpawn(
spawn,
createInputMetadataProvider(
runfilesMiddleman,
runfilesTree,
runfilesMiddleman,
sourceArtifact,
genArtifact,
externalSourceArtifact,
Expand Down Expand Up @@ -830,8 +883,8 @@ public void testRunfilesExternalOnly(
context.logSpawn(
spawn,
createInputMetadataProvider(
runfilesMiddleman,
runfilesTree,
runfilesMiddleman,
externalSourceArtifact,
externalGenArtifact,
symlinkTarget,
Expand Down Expand Up @@ -963,8 +1016,8 @@ public void testRunfilesFilesCollide(@TestParameter boolean legacyExternalRunfil
context.logSpawn(
spawn,
createInputMetadataProvider(
runfilesMiddleman,
runfilesTree,
runfilesMiddleman,
sourceArtifact,
genArtifact,
externalSourceArtifact,
Expand Down Expand Up @@ -1069,8 +1122,8 @@ public void testRunfilesFilesAndSymlinksCollide(@TestParameter boolean legacyExt
context.logSpawn(
spawn,
createInputMetadataProvider(
runfilesMiddleman,
runfilesTree,
runfilesMiddleman,
sourceArtifact,
genArtifact,
externalSourceArtifact,
Expand Down Expand Up @@ -1162,7 +1215,7 @@ public void testRunfilesFileAndRootSymlinkCollide() throws Exception {
context.logSpawn(
spawn,
createInputMetadataProvider(
runfilesMiddleman, runfilesTree, sourceArtifact, symlinkSourceArtifact),
runfilesTree, runfilesMiddleman, sourceArtifact, symlinkSourceArtifact),
createInputMap(runfilesTree),
fs,
defaultTimeout(),
Expand Down Expand Up @@ -1209,7 +1262,7 @@ public void testRunfilesCrossTypeCollision(@TestParameter boolean symlinkFirst)

context.logSpawn(
spawn,
createInputMetadataProvider(runfilesMiddleman, runfilesTree, file, symlink),
createInputMetadataProvider(runfilesTree, runfilesMiddleman, file, symlink),
createInputMap(runfilesTree),
fs,
defaultTimeout(),
Expand Down Expand Up @@ -1286,7 +1339,7 @@ public void testRunfilesPostOrderCollision(@TestParameter boolean nestBoth) thro
context.logSpawn(
spawn,
createInputMetadataProvider(
runfilesMiddleman, runfilesTree, sourceFile, genFile, otherSourceFile, otherGenFile),
runfilesTree, runfilesMiddleman, sourceFile, genFile, otherSourceFile, otherGenFile),
createInputMap(runfilesTree),
fs,
defaultTimeout(),
Expand Down Expand Up @@ -1364,7 +1417,7 @@ public void testRunfilesSymlinkTargets(
context.logSpawn(
spawnBuilder.build(),
createInputMetadataProvider(
runfilesMiddleman, runfilesTree, sourceFile, sourceDir, genDir, symlink),
runfilesTree, runfilesMiddleman, sourceFile, sourceDir, genDir, symlink),
createInputMap(runfilesTree),
fs,
defaultTimeout(),
Expand Down Expand Up @@ -1437,7 +1490,7 @@ public void testRunfileSymlinkFileWithDirectoryContents(

context.logSpawn(
spawn,
createInputMetadataProvider(runfilesMiddleman, runfilesTree, sourceFile),
createInputMetadataProvider(runfilesTree, runfilesMiddleman, sourceFile),
createInputMap(runfilesTree),
outputsMode.getActionFileSystem(fs),
defaultTimeout(),
Expand Down Expand Up @@ -2059,19 +2112,13 @@ protected static RunfilesTree createRunfilesTree(

protected static InputMetadataProvider createInputMetadataProvider(Artifact... artifacts)
throws Exception {
return createInputMetadataProvider(null, null, artifacts);
return createInputMetadataProvider(null, artifacts);
}

protected static InputMetadataProvider createInputMetadataProvider(
Artifact runfilesMiddleman, RunfilesTree runfilesTree, Artifact... artifacts)
throws Exception {
Iterable<Artifact> allArtifacts = Arrays.asList(artifacts);
RunfilesTree runfilesTree, Artifact... artifacts) throws Exception {
FakeActionInputFileCache builder = new FakeActionInputFileCache();
if (runfilesMiddleman != null) {
allArtifacts = Iterables.concat(allArtifacts, runfilesTree.getArtifacts().toList());
builder.putRunfilesTree(runfilesMiddleman, runfilesTree);
}
for (Artifact artifact : allArtifacts) {
for (Artifact artifact : artifacts) {
if (artifact.isTreeArtifact()) {
// Emulate ActionInputMap: add both tree and children.
TreeArtifactValue treeMetadata = createTreeArtifactValue(artifact);
Expand All @@ -2082,6 +2129,8 @@ protected static InputMetadataProvider createInputMetadataProvider(
}
} else if (artifact.isSymlink()) {
builder.put(artifact, FileArtifactValue.createForUnresolvedSymlink(artifact));
} else if (artifact.isMiddlemanArtifact()) {
builder.putRunfilesTree(artifact, runfilesTree);
} else {
builder.put(artifact, FileArtifactValue.createForTesting(artifact));
}
Expand Down

0 comments on commit 07668e6

Please sign in to comment.