From 6c393ec5539b34e9708b43b0145488b9df1c0878 Mon Sep 17 00:00:00 2001 From: George Gensure Date: Mon, 3 Jul 2023 06:17:02 -0700 Subject: [PATCH 01/51] Add temporary rules_go bazel_dep to restore CI https://github.com/bazelbuild/bazel-central-registry/pull/725 broke bazel CI asynchronously by yanking rules_go@0.34.0, listed as a dependency on BCR of grpc@1.48.1. From BCR recommendation on https://github.com/bazelbuild/bazel-central-registry/pull/733, specifying a rules_go@0.39.1 should coerce transitive selection through Minimum Version Selection (and bypass gazelle@0.27.0, also yanked). Closes #18828. PiperOrigin-RevId: 545205842 Change-Id: I574fc46b7c09e72e6d70ef77be726897aacc4358 --- MODULE.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/MODULE.bazel b/MODULE.bazel index 98717d2ffd1319..d912967c8c24cb 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -16,6 +16,7 @@ bazel_dep(name = "stardoc", version = "0.5.3", repo_name = "io_bazel_skydoc") bazel_dep(name = "zstd-jni", version = "1.5.2-3") bazel_dep(name = "zlib", version = "1.2.13") bazel_dep(name = "rules_cc", version = "0.0.6") +bazel_dep(name = "rules_go", version = "0.39.1") bazel_dep(name = "rules_java", version = "6.1.1") bazel_dep(name = "rules_proto", version = "5.3.0-21.7") bazel_dep(name = "rules_jvm_external", version = "5.2") From 32ccdd04c2fae57431264c9a6ce6614038ff192f Mon Sep 17 00:00:00 2001 From: Googler Date: Mon, 3 Jul 2023 07:31:49 -0700 Subject: [PATCH 02/51] Delete unused `JavaInfo` constructor PiperOrigin-RevId: 545217941 Change-Id: I798ca16421b44dfdc9c4cad910ba5d8f3d70156c --- .../build/lib/rules/java/JavaInfo.java | 52 ----- .../lib/rules/java/JavaInfoBuildHelper.java | 102 ---------- .../starlarkbuildapi/java/JavaInfoApi.java | 182 ------------------ 3 files changed, 336 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfo.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfo.java index a2d9939bc80748..adf6d06438bbd8 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfo.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfo.java @@ -36,7 +36,6 @@ import com.google.devtools.build.lib.rules.cpp.LibraryToLink; import com.google.devtools.build.lib.rules.java.JavaPluginInfo.JavaPluginData; import com.google.devtools.build.lib.rules.java.JavaRuleOutputJarsProvider.JavaOutput; -import com.google.devtools.build.lib.starlarkbuildapi.FileApi; import com.google.devtools.build.lib.starlarkbuildapi.cpp.CcInfoApi; import com.google.devtools.build.lib.starlarkbuildapi.java.JavaInfoApi; import com.google.devtools.build.lib.starlarkbuildapi.java.JavaModuleFlagsProviderApi; @@ -52,7 +51,6 @@ import net.starlark.java.eval.Sequence; import net.starlark.java.eval.Starlark; import net.starlark.java.eval.StarlarkList; -import net.starlark.java.eval.StarlarkThread; import net.starlark.java.eval.StarlarkValue; import net.starlark.java.syntax.Location; @@ -647,56 +645,6 @@ private JavaInfoProvider() { super(Label.parseCanonicalUnchecked("@_builtins//:common/java/java_info.bzl"), STARLARK_NAME); } - @Override - public JavaInfo javaInfo( - FileApi outputJarApi, - Object compileJarApi, - Object sourceJarApi, - Object compileJdepsApi, - Object generatedClassJarApi, - Object generatedSourceJarApi, - Object nativeHeadersJarApi, - Object manifestProtoApi, - Boolean neverlink, - Sequence deps, - Sequence runtimeDeps, - Sequence exports, - Sequence exportedPlugins, - Object jdepsApi, - Sequence nativeLibraries, - StarlarkThread thread) - throws EvalException, RuleErrorException { - Artifact outputJar = (Artifact) outputJarApi; - @Nullable Artifact compileJar = nullIfNone(compileJarApi, Artifact.class); - @Nullable Artifact sourceJar = nullIfNone(sourceJarApi, Artifact.class); - @Nullable Artifact compileJdeps = nullIfNone(compileJdepsApi, Artifact.class); - @Nullable Artifact generatedClassJar = nullIfNone(generatedClassJarApi, Artifact.class); - @Nullable Artifact generatedSourceJar = nullIfNone(generatedSourceJarApi, Artifact.class); - @Nullable Artifact nativeHeadersJar = nullIfNone(nativeHeadersJarApi, Artifact.class); - @Nullable Artifact manifestProto = nullIfNone(manifestProtoApi, Artifact.class); - @Nullable Artifact jdeps = nullIfNone(jdepsApi, Artifact.class); - return JavaInfoBuildHelper.getInstance() - .createJavaInfo( - JavaOutput.builder() - .setClassJar(outputJar) - .setCompileJar(compileJar) - .setCompileJdeps(compileJdeps) - .setGeneratedClassJar(generatedClassJar) - .setGeneratedSourceJar(generatedSourceJar) - .setNativeHeadersJar(nativeHeadersJar) - .setManifestProto(manifestProto) - .setJdeps(jdeps) - .addSourceJar(sourceJar) - .build(), - neverlink, - Sequence.cast(deps, JavaInfo.class, "deps"), - Sequence.cast(runtimeDeps, JavaInfo.class, "runtime_deps"), - Sequence.cast(exports, JavaInfo.class, "exports"), - JavaPluginInfo.wrapSequence(exportedPlugins, "exported_plugins"), - Sequence.cast(nativeLibraries, CcInfo.class, "native_libraries"), - thread.getCallerLocation()); - } - @Override public JavaInfo wrap(Info info) throws RuleErrorException { if (info instanceof JavaInfo) { diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfoBuildHelper.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfoBuildHelper.java index 87777b79fdbeda..634a5840e59f15 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfoBuildHelper.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfoBuildHelper.java @@ -16,9 +16,6 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterables.concat; import static com.google.common.collect.Streams.stream; -import static com.google.devtools.build.lib.rules.java.JavaCompilationArgsProvider.ClasspathType.BOTH; -import static com.google.devtools.build.lib.rules.java.JavaCompilationArgsProvider.ClasspathType.COMPILE_ONLY; -import static com.google.devtools.build.lib.rules.java.JavaCompilationArgsProvider.ClasspathType.RUNTIME_ONLY; import static com.google.devtools.build.lib.rules.java.JavaInfo.streamProviders; import static java.util.stream.Stream.concat; @@ -40,8 +37,6 @@ import com.google.devtools.build.lib.collect.nestedset.Order; import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; import com.google.devtools.build.lib.rules.cpp.CcInfo; -import com.google.devtools.build.lib.rules.java.JavaCompilationArgsProvider.ClasspathType; -import com.google.devtools.build.lib.rules.java.JavaRuleOutputJarsProvider.JavaOutput; import com.google.devtools.build.lib.shell.ShellUtils; import com.google.devtools.build.lib.vfs.FileSystemUtils; import java.util.ArrayList; @@ -50,10 +45,8 @@ import java.util.stream.Stream; import javax.annotation.Nullable; import net.starlark.java.eval.EvalException; -import net.starlark.java.eval.Sequence; import net.starlark.java.eval.Starlark; import net.starlark.java.eval.StarlarkThread; -import net.starlark.java.syntax.Location; /** Implements logic for creating JavaInfo from different set of input parameters. */ final class JavaInfoBuildHelper { @@ -65,101 +58,6 @@ public static JavaInfoBuildHelper getInstance() { return INSTANCE; } - /** - * Creates JavaInfo instance from outputJar. - * - * @param javaOutput the artifacts that were created as a result of a compilation (e.g. javac, - * scalac, etc) - * @param neverlink if true only use this library for compilation and not at runtime - * @param compileTimeDeps compile time dependencies that were used to create the output jar - * @param runtimeDeps runtime dependencies that are needed for this library - * @param exports libraries to make available for users of this library. java_library.exports - * @param exportedPlugins A list of exported plugins. - * @param nativeLibraries CC library dependencies that are needed for this library - * @return new created JavaInfo instance - */ - JavaInfo createJavaInfo( - JavaOutput javaOutput, - Boolean neverlink, - Sequence compileTimeDeps, - Sequence runtimeDeps, - Sequence exports, - Iterable exportedPlugins, - Sequence nativeLibraries, - Location location) - throws RuleErrorException, EvalException { - JavaInfo.Builder javaInfoBuilder = JavaInfo.Builder.create(); - javaInfoBuilder.setLocation(location); - javaInfoBuilder.setNeverlink(neverlink); - - JavaCompilationArgsProvider.Builder javaCompilationArgsBuilder = - JavaCompilationArgsProvider.builder(); - - if (!neverlink) { - javaCompilationArgsBuilder.addRuntimeJar(javaOutput.getClassJar()); - } - if (javaOutput.getCompileJar() != null) { - javaCompilationArgsBuilder.addDirectCompileTimeJar( - /* interfaceJar= */ javaOutput.getCompileJar(), /* fullJar= */ javaOutput.getClassJar()); - } - - JavaRuleOutputJarsProvider javaRuleOutputJarsProvider = - JavaRuleOutputJarsProvider.builder().addJavaOutput(javaOutput).build(); - javaInfoBuilder.javaRuleOutputs(javaRuleOutputJarsProvider); - - ClasspathType type = neverlink ? COMPILE_ONLY : BOTH; - - streamProviders(exports, JavaCompilationArgsProvider.class) - .forEach(args -> javaCompilationArgsBuilder.addExports(args, type)); - streamProviders(compileTimeDeps, JavaCompilationArgsProvider.class) - .forEach(args -> javaCompilationArgsBuilder.addDeps(args, type)); - - streamProviders(runtimeDeps, JavaCompilationArgsProvider.class) - .forEach(args -> javaCompilationArgsBuilder.addDeps(args, RUNTIME_ONLY)); - - if (javaOutput.getCompileJdeps() != null) { - javaCompilationArgsBuilder.addCompileTimeJavaDependencyArtifacts( - NestedSetBuilder.create(Order.STABLE_ORDER, javaOutput.getCompileJdeps())); - } - - javaInfoBuilder.javaCompilationArgs(javaCompilationArgsBuilder.build()); - - javaInfoBuilder.javaPluginInfo(mergeExportedJavaPluginInfo(exportedPlugins, exports)); - - javaInfoBuilder.javaSourceJars( - createJavaSourceJarsProvider( - javaOutput.getSourceJars(), concat(compileTimeDeps, runtimeDeps, exports))); - - javaInfoBuilder.javaGenJars( - JavaGenJarsProvider.create( - false, - javaOutput.getGeneratedClassJar(), - javaOutput.getGeneratedSourceJar(), - JavaPluginInfo.empty(), - collectJavaGenJarsProviders(concat(compileTimeDeps, exports)))); - - javaInfoBuilder.setRuntimeJars(ImmutableList.of(javaOutput.getClassJar())); - - ImmutableList transitiveNativeLibraries = - Streams.concat( - streamProviders(runtimeDeps, JavaCcInfoProvider.class), - streamProviders(exports, JavaCcInfoProvider.class), - streamProviders(compileTimeDeps, JavaCcInfoProvider.class), - Stream.of(new JavaCcInfoProvider(CcInfo.merge(nativeLibraries)))) - .collect(toImmutableList()); - javaInfoBuilder.javaCcInfo(JavaCcInfoProvider.merge(transitiveNativeLibraries)); - - javaInfoBuilder.javaModuleFlags( - JavaModuleFlagsProvider.merge( - JavaInfo.streamProviders( - concat(compileTimeDeps, exports), JavaModuleFlagsProvider.class) - .collect(toImmutableList()))); - - return javaInfoBuilder.build(); - } - private static ImmutableList collectJavaGenJarsProviders( Iterable javaInfos) throws RuleErrorException, EvalException { ImmutableList.Builder builder = ImmutableList.builder(); diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaInfoApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaInfoApi.java index 0ceb21f2604687..19cd50c4bda00f 100644 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaInfoApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaInfoApi.java @@ -14,9 +14,7 @@ package com.google.devtools.build.lib.starlarkbuildapi.java; -import com.google.devtools.build.docgen.annot.StarlarkConstructor; import com.google.devtools.build.lib.collect.nestedset.Depset; -import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; import com.google.devtools.build.lib.starlarkbuildapi.FileApi; import com.google.devtools.build.lib.starlarkbuildapi.core.ProviderApi; @@ -24,13 +22,8 @@ import com.google.devtools.build.lib.starlarkbuildapi.cpp.CcInfoApi; import com.google.devtools.build.lib.starlarkbuildapi.java.JavaPluginInfoApi.JavaPluginDataApi; import javax.annotation.Nullable; -import net.starlark.java.annot.Param; -import net.starlark.java.annot.ParamType; import net.starlark.java.annot.StarlarkMethod; -import net.starlark.java.eval.EvalException; -import net.starlark.java.eval.NoneType; import net.starlark.java.eval.Sequence; -import net.starlark.java.eval.StarlarkThread; /** Info object encapsulating all information by java rules. */ public interface JavaInfoApi< @@ -181,180 +174,5 @@ public interface JavaInfoApi< /** Provider class for {@link JavaInfoApi} objects. */ interface JavaInfoProviderApi extends ProviderApi { - @StarlarkMethod( - name = "JavaInfo", - doc = "The JavaInfo constructor.", - parameters = { - @Param( - name = "output_jar", - named = true, - doc = - "The jar that was created as a result of a compilation " - + "(e.g. javac, scalac, etc)."), - @Param( - name = "compile_jar", - allowedTypes = { - @ParamType(type = FileApi.class), - @ParamType(type = NoneType.class), - }, - named = true, - doc = - "A jar that is added as the compile-time dependency in lieu of " - + "output_jar. Typically this is the ijar produced by " - + "" - + "run_ijar. " - + "If you cannot use ijar, consider instead using the output of " - + "" - + "stamp_ijar. If you do not wish to use either, " - + "you can simply pass output_jar. " - + "There are a couple of special cases when this parameter may be set to " - + "None, for example adding a jar with resources or when used in" - + " a terminal rule like java_binary."), - @Param( - name = "source_jar", - allowedTypes = { - @ParamType(type = FileApi.class), - @ParamType(type = NoneType.class), - }, - named = true, - defaultValue = "None", - doc = - "The source jar that was used to create the output jar. Use pack_sources" - + " to produce this source jar."), - @Param( - name = "compile_jdeps", - allowedTypes = { - @ParamType(type = FileApi.class), - @ParamType(type = NoneType.class), - }, - named = true, - defaultValue = "None", - doc = - "jdeps information about compile time dependencies to be consumed by" - + " JavaCompileAction. This should be a binary proto encoded using the" - + " deps.proto protobuf included with Bazel. If available this file is" - + " typically produced by a header compiler."), - @Param( - name = "generated_class_jar", - allowedTypes = { - @ParamType(type = FileApi.class), - @ParamType(type = NoneType.class), - }, - named = true, - defaultValue = "None", - doc = - "A jar file containing class files compiled from sources generated during" - + " annotation processing."), - @Param( - name = "generated_source_jar", - allowedTypes = { - @ParamType(type = FileApi.class), - @ParamType(type = NoneType.class), - }, - named = true, - defaultValue = "None", - doc = "The source jar that was created as a result of annotation processing."), - @Param( - name = "native_headers_jar", - allowedTypes = { - @ParamType(type = FileApi.class), - @ParamType(type = NoneType.class), - }, - named = true, - defaultValue = "None", - doc = - "A jar containing CC header files supporting native method implementation" - + " (typically output of javac -h)."), - @Param( - name = "manifest_proto", - allowedTypes = { - @ParamType(type = FileApi.class), - @ParamType(type = NoneType.class), - }, - named = true, - defaultValue = "None", - doc = - "Manifest information for the rule output (if available). This should be a" - + " binary proto encoded using the manifest.proto protobuf included with" - + " Bazel. IDEs and other tools can use this information for more efficient" - + " processing."), - @Param( - name = "neverlink", - named = true, - defaultValue = "False", - doc = "If true only use this library for compilation and not at runtime."), - @Param( - name = "deps", - allowedTypes = {@ParamType(type = Sequence.class, generic1 = JavaInfoApi.class)}, - named = true, - defaultValue = "[]", - doc = "Compile time dependencies that were used to create the output jar."), - @Param( - name = "runtime_deps", - allowedTypes = {@ParamType(type = Sequence.class, generic1 = JavaInfoApi.class)}, - named = true, - defaultValue = "[]", - doc = "Runtime dependencies that are needed for this library."), - @Param( - name = "exports", - allowedTypes = {@ParamType(type = Sequence.class, generic1 = JavaInfoApi.class)}, - named = true, - defaultValue = "[]", - doc = - "Libraries to make available for users of this library. See also " - + "java_library.exports."), - @Param( - name = "exported_plugins", - named = true, - allowedTypes = { - @ParamType(type = Sequence.class, generic1 = JavaPluginInfoApi.class) - }, - defaultValue = "[]", - doc = "A list of exported plugins. Optional."), - @Param( - name = "jdeps", - allowedTypes = { - @ParamType(type = FileApi.class), - @ParamType(type = NoneType.class), - }, - named = true, - defaultValue = "None", - doc = - "jdeps information for the rule output (if available). This should be a binary" - + " proto encoded using the deps.proto protobuf included with Bazel. If" - + " available this file is typically produced by a compiler. IDEs and other" - + " tools can use this information for more efficient processing."), - @Param( - name = "native_libraries", - allowedTypes = {@ParamType(type = Sequence.class, generic1 = CcInfoApi.class)}, - named = true, - defaultValue = "[]", - doc = "CC native library dependencies that are needed for this library."), - }, - selfCall = true, - useStarlarkThread = true) - @StarlarkConstructor - JavaInfoApi javaInfo( - FileApi outputJarApi, - Object compileJarApi, - Object sourceJarApi, - Object compileJdepsApi, - Object generatedClassJarApi, - Object generatedSourceJarApi, - Object nativeHeadersJarApi, - Object manifestProtoApi, - Boolean neverlink, - Sequence deps, - Sequence runtimeDeps, - Sequence exports, - Sequence exportedPlugins, - Object jdepsApi, - Sequence nativeLibraries, - StarlarkThread thread) - throws EvalException, RuleErrorException; } } From cf7e6df115abfa26790f762459a1932edf746a71 Mon Sep 17 00:00:00 2001 From: Googler Date: Mon, 3 Jul 2023 07:34:13 -0700 Subject: [PATCH 03/51] Add old name warning for `experimental_worker_max_multiplex_instances` and `experimental_worker_multiplex`. PiperOrigin-RevId: 545218355 Change-Id: Ie25a94365808d5b2067442b75ac9adb0affed00d --- .../com/google/devtools/build/lib/worker/WorkerOptions.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerOptions.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerOptions.java index dbb96c833c95f9..35158766aece1d 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/WorkerOptions.java +++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerOptions.java @@ -94,7 +94,6 @@ public String getTypeDescription() { @Option( name = "worker_max_multiplex_instances", oldName = "experimental_worker_max_multiplex_instances", - oldNameWarning = false, converter = MultiResourceConverter.class, defaultValue = "null", documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY, @@ -151,7 +150,6 @@ public String getTypeDescription() { @Option( name = "worker_multiplex", oldName = "experimental_worker_multiplex", - oldNameWarning = false, defaultValue = "true", documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY, effectTags = {OptionEffectTag.EXECUTION, OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS}, From 05e3b7b5aabc0eaaa7ccf9a9a80974853f5811e0 Mon Sep 17 00:00:00 2001 From: Googler Date: Mon, 3 Jul 2023 08:13:35 -0700 Subject: [PATCH 04/51] Make grep_includes optional in cc_common PiperOrigin-RevId: 545225277 Change-Id: I20c267cc3fd999be1d00986a4cca97df15bc0bf1 --- src/main/starlark/builtins_bzl/common/cc/cc_common.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/starlark/builtins_bzl/common/cc/cc_common.bzl b/src/main/starlark/builtins_bzl/common/cc/cc_common.bzl index b2375707d18443..2291c46fffb36e 100644 --- a/src/main/starlark/builtins_bzl/common/cc/cc_common.bzl +++ b/src/main/starlark/builtins_bzl/common/cc/cc_common.bzl @@ -589,13 +589,13 @@ def _register_linkstamp_compile_action( actions, cc_toolchain, feature_configuration, - grep_includes, source_file, output_file, compilation_inputs, inputs_for_validation, label_replacement, - output_replacement): + output_replacement, + grep_includes = None): cc_common_internal.check_private_api(allowlist = _PRIVATE_STARLARKIFICATION_ALLOWLIST) return cc_common_internal.register_linkstamp_compile_action( actions = actions, @@ -786,13 +786,13 @@ def _create_compile_action( actions, cc_toolchain, feature_configuration, - grep_includes, source_file, output_file, variables, action_name, compilation_context, additional_inputs = None, + grep_includes = None, additional_outputs = []): cc_common_internal.check_private_api(allowlist = _CREATE_COMPILE_ACTION_API_ALLOWLISTED_PACKAGES) return cc_common_internal.create_compile_action( From f706e7d5da8f715198b69b21045e2cb786e70bec Mon Sep 17 00:00:00 2001 From: Googler Date: Mon, 3 Jul 2023 08:22:57 -0700 Subject: [PATCH 05/51] Enable Starlark `java_common` PiperOrigin-RevId: 545226950 Change-Id: Ie5e3ecac5a64157ea210acfd61be160bf1da3d90 --- .../java/bazel_java_binary_deploy_jar.bzl | 2 +- .../common/java/basic_java_library.bzl | 3 +- .../common/java/compile_action.bzl | 8 +- .../builtins_bzl/common/java/java_binary.bzl | 4 +- .../builtins_bzl/common/java/java_common.bzl | 239 +++++++++++++--- .../java_common_internal_for_builtins.bzl | 265 ++++++++++++++++++ .../builtins_bzl/common/java/java_import.bzl | 3 +- .../common/java/java_plugin_info.bzl | 6 +- .../common/java/java_semantics.bzl | 1 + .../common/java/proto/java_proto_library.bzl | 16 +- .../lib/rules/java/JavaStarlarkApiTest.java | 53 +--- 11 files changed, 508 insertions(+), 92 deletions(-) create mode 100644 src/main/starlark/builtins_bzl/common/java/java_common_internal_for_builtins.bzl diff --git a/src/main/starlark/builtins_bzl/bazel/java/bazel_java_binary_deploy_jar.bzl b/src/main/starlark/builtins_bzl/bazel/java/bazel_java_binary_deploy_jar.bzl index bed4a2de7a38db..204de369a57a93 100644 --- a/src/main/starlark/builtins_bzl/bazel/java/bazel_java_binary_deploy_jar.bzl +++ b/src/main/starlark/builtins_bzl/bazel/java/bazel_java_binary_deploy_jar.bzl @@ -20,7 +20,7 @@ the generating actions, so that the runfiles symlink tree is staged for the depl load(":common/java/java_binary_deploy_jar.bzl", "create_deploy_archives", "make_deploy_jars_rule") load(":common/java/java_binary.bzl", "InternalDeployJarInfo") -load(":common/java/java_common.bzl", "get_build_info") +load(":common/java/java_common_internal_for_builtins.bzl", "get_build_info") def _stamping_enabled(ctx, stamp): if ctx.configuration.is_tool_configuration(): diff --git a/src/main/starlark/builtins_bzl/common/java/basic_java_library.bzl b/src/main/starlark/builtins_bzl/common/java/basic_java_library.bzl index d1c6c781a80a45..bb6d9c4c2d8056 100644 --- a/src/main/starlark/builtins_bzl/common/java/basic_java_library.bzl +++ b/src/main/starlark/builtins_bzl/common/java/basic_java_library.bzl @@ -24,7 +24,8 @@ load(":common/java/proguard_validation.bzl", "validate_proguard_specs") load(":common/cc/cc_info.bzl", "CcInfo") load(":common/java/java_info.bzl", "JavaInfo") load(":common/java/java_plugin_info.bzl", "JavaPluginInfo") -load(":common/java/java_common.bzl", "java_common", "target_kind") +load(":common/java/java_common.bzl", "java_common") +load(":common/java/java_common_internal_for_builtins.bzl", "target_kind") coverage_common = _builtins.toplevel.coverage_common diff --git a/src/main/starlark/builtins_bzl/common/java/compile_action.bzl b/src/main/starlark/builtins_bzl/common/java/compile_action.bzl index 0e4e9a36d533b7..fdc29b5f6aa5d8 100644 --- a/src/main/starlark/builtins_bzl/common/java/compile_action.bzl +++ b/src/main/starlark/builtins_bzl/common/java/compile_action.bzl @@ -17,7 +17,7 @@ Java compile action """ load(":common/java/java_semantics.bzl", "semantics") -load(":common/java/java_common.bzl", "java_common") +load(":common/java/java_common_internal_for_builtins.bzl", _compile_private_for_builtins = "compile") def _filter_strict_deps(mode): return "error" if mode in ["strict", "default"] else mode @@ -130,8 +130,10 @@ def compile_action( or resources present, whereas runfiles in this case are empty. """ - java_info = java_common.compile( + java_info = _compile_private_for_builtins( ctx, + output = output_class_jar, + java_toolchain = semantics.find_java_toolchain(ctx), source_files = source_files, source_jars = source_jars, resources = resources, @@ -145,8 +147,6 @@ def compile_action( exported_plugins = exported_plugins, javac_opts = [ctx.expand_location(opt) for opt in javacopts], neverlink = neverlink, - java_toolchain = semantics.find_java_toolchain(ctx), - output = output_class_jar, output_source_jar = output_source_jar, strict_deps = _filter_strict_deps(strict_deps), enable_compile_jar_action = enable_compile_jar_action, diff --git a/src/main/starlark/builtins_bzl/common/java/java_binary.bzl b/src/main/starlark/builtins_bzl/common/java/java_binary.bzl index 8e3e924b293dc0..1939d91f1fcc1b 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_binary.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_binary.bzl @@ -24,11 +24,11 @@ load(":common/cc/cc_info.bzl", "CcInfo") load(":common/paths.bzl", "paths") load(":common/java/java_info.bzl", "JavaInfo") load(":common/java/java_plugin_info.bzl", "JavaPluginInfo") +load(":common/java/java_common.bzl", "java_common") load( - ":common/java/java_common.bzl", + ":common/java/java_common_internal_for_builtins.bzl", "collect_native_deps_dirs", "get_runtime_classpath_for_archive", - "java_common", "to_java_binary_info", ) diff --git a/src/main/starlark/builtins_bzl/common/java/java_common.bzl b/src/main/starlark/builtins_bzl/common/java/java_common.bzl index 6c8ae716aca504..21b9ecd03eb06e 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_common.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_common.bzl @@ -14,71 +14,238 @@ """ Utilities for Java compilation support in Starlark. """ +load(":common/java/java_info.bzl", "JavaInfo") +load(":common/java/java_common_internal_for_builtins.bzl", "compile", "merge", "run_ijar") +load(":common/java/java_plugin_info.bzl", "JavaPluginInfo") +load(":common/java/java_semantics.bzl", "semantics") + _java_common_internal = _builtins.internal.java_common_internal_do_not_use -def target_kind(target, dereference_aliases = False): - """ Get the rule class string for a target +def _compile( + ctx, + output, + java_toolchain, + source_jars = [], + source_files = [], + output_source_jar = None, + javac_opts = [], + deps = [], + runtime_deps = [], + exports = [], + plugins = [], + exported_plugins = [], + native_libraries = [], + annotation_processor_additional_inputs = [], + annotation_processor_additional_outputs = [], + strict_deps = "ERROR", + bootclasspath = None, + sourcepath = [], + resources = [], + neverlink = False, + enable_annotation_processing = True, + add_exports = [], + add_opens = []): + return compile( + ctx, + output, + java_toolchain, + source_jars = source_jars, + source_files = source_files, + output_source_jar = output_source_jar, + javac_opts = javac_opts, + deps = deps, + runtime_deps = runtime_deps, + exports = exports, + plugins = plugins, + exported_plugins = exported_plugins, + native_libraries = native_libraries, + annotation_processor_additional_inputs = annotation_processor_additional_inputs, + annotation_processor_additional_outputs = annotation_processor_additional_outputs, + strict_deps = strict_deps, + bootclasspath = bootclasspath, + sourcepath = sourcepath, + resources = resources, + neverlink = neverlink, + enable_annotation_processing = enable_annotation_processing, + add_exports = add_exports, + add_opens = add_opens, + ) + +def _run_ijar(actions, jar, java_toolchain, target_label = None): + return run_ijar( + actions = actions, + jar = jar, + java_toolchain = java_toolchain, + target_label = target_label, + ) + +def _stamp_jar(actions, jar, java_toolchain, target_label): + """Stamps a jar with a target label for add_dep support. + + The return value is typically passed to `JavaInfo.compile_jar`. Prefer to use `run_ijar` when + possible. Args: - target: (Target) - dereference_aliases: (bool) resolve the actual target rule class if an - alias + actions: (actions) ctx.actions + jar: (File) The jar to run stamp_jar on. + java_toolchain: (JavaToolchainInfo) The toolchain to used to find the stamp_jar tool. + target_label: (Label) A target label to stamp the jar with. Used for `add_dep` support. + Typically, you would pass `ctx.label` to stamp the jar with the current rule's label. Returns: - (str) The rule class string of the target + (File) The output artifact + """ - return _java_common_internal.target_kind( - target, - dereference_aliases = dereference_aliases, + return _java_common_internal.stamp_jar( + actions = actions, + jar = jar, + java_toolchain = java_toolchain, + target_label = target_label, ) -def to_java_binary_info(java_info): - """ Get a copy of the given JavaInfo with minimal info returned by a java_binary +def _pack_sources( + actions, + java_toolchain, + output_source_jar = None, + sources = [], + source_jars = []): + """Packs sources and source jars into a single source jar file. + + The return value is typically passed to `JavaInfo.source_jar`. At least one of parameters + output_jar or output_source_jar is required. Args: - java_info: (JavaInfo) A JavaInfo provider instance + actions: (actions) ctx.actions + java_toolchain: (JavaToolchainInfo) The toolchain used to find the ijar tool. + output_source_jar: (File) The output source jar. + sources: ([File]) A list of Java source files to be packed into the source jar. + source_jars: ([File]) A list of source jars to be packed into the source jar. Returns: - (JavaInfo) A JavaInfo instance representing a java_binary target + (File) The output artifact """ - return _java_common_internal.to_java_binary_info(java_info) + return _java_common_internal.pack_sources( + actions = actions, + java_toolchain = java_toolchain, + sources = sources, + source_jars = source_jars, + output_source_jar = output_source_jar, + ) -def get_build_info(ctx, is_stamping_enabled): - """ Get the artifacts representing the workspace status for this build +def _default_javac_opts(java_toolchain): + """Experimental! Get default javacopts from a java toolchain Args: - ctx: (RuleContext) The rule context - is_stamping_enabled: (bool) If stamping is enabled + java_toolchain: (JavaToolchainInfo) the toolchain from which to get the javac options. - Returns - ([File]) The build info artifacts + Returns: + ([str]) A list of javac options """ - return _java_common_internal.get_build_info(ctx, is_stamping_enabled) + return _java_common_internal.default_javac_opts(java_toolchain = java_toolchain) -def collect_native_deps_dirs(deps): - """ Collect the set of root-relative paths containing native libraries +def _merge(providers): + return merge(providers) + +def _make_non_strict(java_info): + """Returns a new JavaInfo instance whose direct-jars part is the union of both the direct and indirect jars of the given Java provider. Args: - deps: [Target] list of targets + java_info: (JavaInfo) The java info to make non-strict. Returns: - ([String]) A set of root-relative paths as a list + (JavaInfo) """ - return _java_common_internal.collect_native_deps_dirs(deps) + return _java_common_internal.make_non_strict(java_info) -def get_runtime_classpath_for_archive(jars, excluded_jars): - """ Filters a classpath to remove certain entries +def _get_message_bundle_info(): + return None if semantics.IS_BAZEL else _java_common_internal.MessageBundleInfo - Args - jars: (depset[File]) The classpath to filter - excluded_jars: (depset[File]) The files to remove +def _add_constraints(java_info, constraints = []): + """Returns a copy of the given JavaInfo with the given constraints added. + + Args: + java_info: (JavaInfo) The JavaInfo to enhance + constraints: ([str]) Constraints to add Returns: - (depset[File]) The filtered classpath + (JavaInfo) """ - return _java_common_internal.get_runtime_classpath_for_archive( - jars, - excluded_jars, + return _java_common_internal.add_constraints(java_info, constraints = constraints) + +def _get_constraints(java_info): + """Returns a set of constraints added. + + Args: + java_info: (JavaInfo) The JavaInfo to get constraints from. + + Returns: + ([str]) The constraints set on the supplied JavaInfo + """ + return _java_common_internal.get_constraints(java_info) + +def _set_annotation_processing( + java_info, + enabled = False, + processor_classnames = [], + processor_classpath = None, + class_jar = None, + source_jar = None): + """Returns a copy of the given JavaInfo with the given annotation_processing info. + + Args: + java_info: (JavaInfo) The JavaInfo to enhance. + enabled: (bool) Whether the rule uses annotation processing. + processor_classnames: ([str]) Class names of annotation processors applied. + processor_classpath: (depset[File]) Class names of annotation processors applied. + class_jar: (File) Optional. Jar that is the result of annotation processing. + source_jar: (File) Optional. Source archive resulting from annotation processing. + + Returns: + (JavaInfo) + """ + return _java_common_internal.set_annotation_processing( + java_info, + enabled = enabled, + processor_classnames = processor_classnames, + processor_classpath = processor_classpath, + class_jar = class_jar, + source_jar = source_jar, ) -java_common = _java_common_internal +def _java_toolchain_label(java_toolchain): + """Returns the toolchain's label. + + Args: + java_toolchain: (JavaToolchainInfo) The toolchain. + Returns: + (Label) + """ + return _java_common_internal.java_toolchain_label(java_toolchain) + +def _make_java_common(): + methods = { + "provider": JavaInfo, + "compile": _compile, + "run_ijar": _run_ijar, + "stamp_jar": _stamp_jar, + "pack_sources": _pack_sources, + "default_javac_opts": _default_javac_opts, + "merge": _merge, + "make_non_strict": _make_non_strict, + "JavaPluginInfo": JavaPluginInfo, + "JavaToolchainInfo": _java_common_internal.JavaToolchainInfo, + "JavaRuntimeInfo": _java_common_internal.JavaRuntimeInfo, + "BootClassPathInfo": _java_common_internal.BootClassPathInfo, + "experimental_java_proto_library_default_has_services": _java_common_internal.experimental_java_proto_library_default_has_services, + } + if _java_common_internal._google_legacy_api_enabled(): + methods.update( + MessageBundleInfo = _get_message_bundle_info(), # struct field that is None in bazel + add_constraints = _add_constraints, + get_constraints = _get_constraints, + set_annotation_processing = _set_annotation_processing, + java_toolchain_label = _java_toolchain_label, + ) + return struct(**methods) + +java_common = _make_java_common() diff --git a/src/main/starlark/builtins_bzl/common/java/java_common_internal_for_builtins.bzl b/src/main/starlark/builtins_bzl/common/java/java_common_internal_for_builtins.bzl new file mode 100644 index 00000000000000..63484016de6307 --- /dev/null +++ b/src/main/starlark/builtins_bzl/common/java/java_common_internal_for_builtins.bzl @@ -0,0 +1,265 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Private utilities for Java compilation support in Starlark. """ + +_java_common_internal = _builtins.internal.java_common_internal_do_not_use + +def compile( + ctx, + output, + java_toolchain, + source_jars = [], + source_files = [], + output_source_jar = None, + javac_opts = [], + deps = [], + runtime_deps = [], + exports = [], + plugins = [], + exported_plugins = [], + native_libraries = [], + annotation_processor_additional_inputs = [], + annotation_processor_additional_outputs = [], + strict_deps = "ERROR", + bootclasspath = None, + sourcepath = [], + resources = [], + add_exports = [], + add_opens = [], + neverlink = False, + enable_annotation_processing = True, + # private to @_builtins: + enable_compile_jar_action = True, + enable_jspecify = True, + include_compilation_info = True, + classpath_resources = [], + resource_jars = [], + injecting_rule_kind = None): + """Compiles Java source files/jars from the implementation of a Starlark rule + + The result is a provider that represents the results of the compilation and can be added to the + set of providers emitted by this rule. + + Args: + ctx: (RuleContext) The rule context + output: (File) The output of compilation + java_toolchain: (JavaToolchainInfo) Toolchain to be used for this compilation. Mandatory. + source_jars: ([File]) A list of the jars to be compiled. At least one of source_jars or + source_files should be specified. + source_files: ([File]) A list of the Java source files to be compiled. At least one of + source_jars or source_files should be specified. + output_source_jar: (File) The output source jar. Optional. Defaults to + `{output_jar}-src.jar` if unset. + javac_opts: ([str]) A list of the desired javac options. Optional. + deps: ([JavaInfo]) A list of dependencies. Optional. + runtime_deps: ([JavaInfo]) A list of runtime dependencies. Optional. + exports: ([JavaInfo]) A list of exports. Optional. + plugins: ([JavaPluginInfo|JavaInfo]) A list of plugins. Optional. + exported_plugins: ([JavaPluginInfo|JavaInfo]) A list of exported plugins. Optional. + native_libraries: ([CcInfo]) CC library dependencies that are needed for this library. + annotation_processor_additional_inputs: ([File]) A list of inputs that the Java compilation + action will take in addition to the Java sources for annotation processing. + annotation_processor_additional_outputs: ([File]) A list of outputs that the Java + compilation action will output in addition to the class jar from annotation processing. + strict_deps: (str) A string that specifies how to handle strict deps. Possible values: + 'OFF', 'ERROR', 'WARN' and 'DEFAULT'. + bootclasspath: (BootClassPathInfo) If present, overrides the bootclasspath associated with + the provided java_toolchain. Optional. + sourcepath: ([File]) + resources: ([File]) + resource_jars: ([File]) + classpath_resources: ([File]) + neverlink: (bool) + enable_annotation_processing: (bool) Disables annotation processing in this compilation, + causing any annotation processors provided in plugins or in exported_plugins of deps to + be ignored. + enable_compile_jar_action: (bool) Enables header compilation or ijar creation. If set to + False, it forces use of the full class jar in the compilation classpaths of any + dependants. Doing so is intended for use by non-library targets such as binaries that + do not have dependants. + enable_jspecify: (bool) + include_compilation_info: (bool) + injecting_rule_kind: (str|None) + add_exports: ([str]) Allow this library to access the given /. Optional. + add_opens: ([str]) Allow this library to reflectively access the given /. + Optional. + + Returns: + (JavaInfo) + """ + return _java_common_internal.compile( + ctx, + output = output, + java_toolchain = java_toolchain, + source_jars = source_jars, + source_files = source_files, + output_source_jar = output_source_jar, + javac_opts = javac_opts, + deps = deps, + runtime_deps = runtime_deps, + exports = exports, + plugins = plugins, + exported_plugins = exported_plugins, + native_libraries = native_libraries, + annotation_processor_additional_inputs = annotation_processor_additional_inputs, + annotation_processor_additional_outputs = annotation_processor_additional_outputs, + strict_deps = strict_deps, + bootclasspath = bootclasspath, + sourcepath = sourcepath, + resources = resources, + neverlink = neverlink, + enable_annotation_processing = enable_annotation_processing, + add_exports = add_exports, + add_opens = add_opens, + enable_compile_jar_action = enable_compile_jar_action, + enable_jspecify = enable_jspecify, + include_compilation_info = include_compilation_info, + classpath_resources = classpath_resources, + resource_jars = resource_jars, + injecting_rule_kind = injecting_rule_kind, + ) + +def run_ijar( + actions, + jar, + java_toolchain, + target_label = None, + # private to @_builtins: + output = None): + """Runs ijar on a jar, stripping it of its method bodies. + + This helps reduce rebuilding of dependent jars during any recompiles consisting only of simple + changes to method implementations. The return value is typically passed to JavaInfo.compile_jar + + Args: + actions: (actions) ctx.actions + jar: (File) The jar to run ijar on. + java_toolchain: (JavaToolchainInfo) The toolchain to used to find the ijar tool. + target_label: (Label|None) A target label to stamp the jar with. Used for `add_dep` support. + Typically, you would pass `ctx.label` to stamp the jar with the current rule's label. + output: (File) Optional. + + Returns: + (File) The output artifact + """ + return _java_common_internal.run_ijar( + actions = actions, + jar = jar, + java_toolchain = java_toolchain, + target_label = target_label, + output = output, + ) + +def merge( + providers, + # private to @_builtins: + merge_java_outputs = True, + merge_source_jars = True): + """Merges the given providers into a single JavaInfo. + + Args: + providers: ([JavaInfo]) The list of providers to merge. + merge_java_outputs: (bool) + merge_source_jars: (bool) + + Returns: + (JavaInfo) The merged JavaInfo + """ + return _java_common_internal.merge( + providers, + merge_java_outputs = merge_java_outputs, + merge_source_jars = merge_source_jars, + ) + +def target_kind(target, dereference_aliases = False): + """Get the rule class string for a target + + Args: + target: (Target) + dereference_aliases: (bool) resolve the actual target rule class if an + alias + + Returns: + (str) The rule class string of the target + """ + return _java_common_internal.target_kind( + target, + dereference_aliases = dereference_aliases, + ) + +def to_java_binary_info(java_info): + """Get a copy of the given JavaInfo with minimal info returned by a java_binary + + Args: + java_info: (JavaInfo) A JavaInfo provider instance + + Returns: + (JavaInfo) A JavaInfo instance representing a java_binary target + """ + return _java_common_internal.to_java_binary_info(java_info) + +def get_build_info(ctx, is_stamping_enabled): + """Get the artifacts representing the workspace status for this build + + Args: + ctx: (RuleContext) The rule context + is_stamping_enabled: (bool) If stamping is enabled + + Returns + ([File]) The build info artifacts + """ + return _java_common_internal.get_build_info(ctx, is_stamping_enabled) + +def collect_native_deps_dirs(deps): + """Collect the set of root-relative paths containing native libraries + + Args: + deps: [Target] list of targets + + Returns: + ([String]) A set of root-relative paths as a list + """ + return _java_common_internal.collect_native_deps_dirs(deps) + +def get_runtime_classpath_for_archive(jars, excluded_jars): + """Filters a classpath to remove certain entries + + Args + jars: (depset[File]) The classpath to filter + excluded_jars: (depset[File]) The files to remove + + Returns: + (depset[File]) The filtered classpath + """ + return _java_common_internal.get_runtime_classpath_for_archive( + jars, + excluded_jars, + ) + +def filter_protos_for_generated_extension_registry(runtime_jars, deploy_env): + """Get proto artifacts from runtime_jars excluding those in deploy_env + + Args: + runtime_jars: (depset[File]) the artifacts to scan + deploy_env: (depset[File]) the artifacts to exclude + + Returns + (depset[File], bool) A tuple of the filtered protos and whether all protos are 'lite' + flavored + """ + return _java_common_internal.filter_protos_for_generated_extension_registry( + runtime_jars, + deploy_env, + ) diff --git a/src/main/starlark/builtins_bzl/common/java/java_import.bzl b/src/main/starlark/builtins_bzl/common/java/java_import.bzl index 6aa2e850504fd3..6dce021f97dc39 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_import.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_import.bzl @@ -23,6 +23,7 @@ load(":common/java/import_deps_check.bzl", "import_deps_check") load(":common/cc/cc_info.bzl", "CcInfo") load(":common/java/java_info.bzl", "JavaInfo") load(":common/java/java_common.bzl", "java_common") +load(":common/java/java_common_internal_for_builtins.bzl", _run_ijar_private_for_builtins = "run_ijar") def _filter_provider(provider, *attrs): return [dep[provider] for attr in attrs for dep in attr if provider in dep] @@ -49,7 +50,7 @@ def _process_with_ijars_if_needed(jars, ctx): interface_jar_directory = "_ijar/" + ctx.label.name + "/" + ijar_basename interface_jar = ctx.actions.declare_file(interface_jar_directory) - java_common.run_ijar( + _run_ijar_private_for_builtins( ctx.actions, target_label = ctx.label, jar = jar, diff --git a/src/main/starlark/builtins_bzl/common/java/java_plugin_info.bzl b/src/main/starlark/builtins_bzl/common/java/java_plugin_info.bzl index 7c0f79fb0cd6be..8743af38e8edda 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_plugin_info.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_plugin_info.bzl @@ -16,7 +16,7 @@ Definition of JavaPluginInfo provider. """ -load(":common/java/java_common.bzl", "java_common") +load(":common/java/java_common_internal_for_builtins.bzl", _merge_private_for_builtins = "merge") _JavaPluginDataInfo = provider( doc = "Provider encapsulating information about a Java compatible plugin.", @@ -58,7 +58,9 @@ def _javaplugininfo_init( Returns: (JavaPluginInfo) """ - java_infos = java_common.merge(runtime_deps) + + # we don't need the private API but java_common needs JavaPluginInfo which would be a cycle + java_infos = _merge_private_for_builtins(runtime_deps) processor_data = data if type(data) == "depset" else depset(data) plugins = _JavaPluginDataInfo( processor_classes = depset([processor_class]) if processor_class else depset(), diff --git a/src/main/starlark/builtins_bzl/common/java/java_semantics.bzl b/src/main/starlark/builtins_bzl/common/java/java_semantics.bzl index a13b1404e6ff10..07686bedb09e06 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_semantics.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_semantics.bzl @@ -57,4 +57,5 @@ semantics = struct( JAVA_STUB_TEMPLATE_LABEL = "@bazel_tools//tools/jdk:java_stub_template.txt", BUILD_INFO_TRANSLATOR_LABEL = None, JAVA_TEST_RUNNER_LABEL = "@bazel_tools//tools/jdk:TestRunner", + IS_BAZEL = True, ) diff --git a/src/main/starlark/builtins_bzl/common/java/proto/java_proto_library.bzl b/src/main/starlark/builtins_bzl/common/java/proto/java_proto_library.bzl index f3b557dd6528de..c4ed89c0e187d1 100644 --- a/src/main/starlark/builtins_bzl/common/java/proto/java_proto_library.bzl +++ b/src/main/starlark/builtins_bzl/common/java/proto/java_proto_library.bzl @@ -18,7 +18,11 @@ load(":common/java/java_semantics.bzl", "semantics") load(":common/proto/proto_common.bzl", "ProtoLangToolchainInfo", proto_common = "proto_common_do_not_use") load(":common/proto/proto_info.bzl", "ProtoInfo") load(":common/java/java_info.bzl", "JavaInfo") -load(":common/java/java_common.bzl", "java_common") +load( + ":common/java/java_common_internal_for_builtins.bzl", + _compile_private_for_builtins = "compile", + _merge_private_for_builtins = "merge", +) # The provider is used to collect source and runtime jars in the `proto_library` dependency graph. JavaProtoAspectInfo = provider("JavaProtoAspectInfo", fields = ["jars"]) @@ -106,23 +110,23 @@ def java_compile_for_protos(ctx, output_jar_suffix, source_jar = None, deps = [] path, sep, filename = ctx.label.name.rpartition("/") output_jar = ctx.actions.declare_file(path + sep + "lib" + filename + output_jar_suffix) java_toolchain = semantics.find_java_toolchain(ctx) - java_info = java_common.compile( + java_info = _compile_private_for_builtins( ctx, + output = output_jar, + java_toolchain = java_toolchain, source_jars = [source_jar], deps = deps, exports = exports, - output = output_jar, output_source_jar = source_jar, injecting_rule_kind = injecting_rule_kind, javac_opts = java_toolchain.compatible_javacopts("proto"), enable_jspecify = False, - java_toolchain = java_toolchain, include_compilation_info = False, ) jars = [source_jar, output_jar] else: # If there are no proto sources just pass along the compilation dependencies. - java_info = java_common.merge(deps + exports, merge_java_outputs = False, merge_source_jars = False) + java_info = _merge_private_for_builtins(deps + exports, merge_java_outputs = False, merge_source_jars = False) jars = [] return java_info, jars @@ -149,7 +153,7 @@ def bazel_java_proto_library_rule(ctx): ([JavaInfo, DefaultInfo, OutputGroupInfo]) """ - java_info = java_common.merge([dep[JavaInfo] for dep in ctx.attr.deps], merge_java_outputs = False) + java_info = _merge_private_for_builtins([dep[JavaInfo] for dep in ctx.attr.deps], merge_java_outputs = False) transitive_src_and_runtime_jars = depset(transitive = [dep[JavaProtoAspectInfo].jars for dep in ctx.attr.deps]) transitive_runtime_jars = depset(transitive = [java_info.transitive_runtime_jars]) diff --git a/src/test/java/com/google/devtools/build/lib/rules/java/JavaStarlarkApiTest.java b/src/test/java/com/google/devtools/build/lib/rules/java/JavaStarlarkApiTest.java index a9544944a49a43..579eaf8c695c03 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/java/JavaStarlarkApiTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/java/JavaStarlarkApiTest.java @@ -29,7 +29,6 @@ import com.google.devtools.build.lib.actions.CommandLineExpansionException; import com.google.devtools.build.lib.actions.util.ActionsTestUtil; import com.google.devtools.build.lib.analysis.ConfiguredTarget; -import com.google.devtools.build.lib.analysis.FileProvider; import com.google.devtools.build.lib.analysis.actions.SpawnAction; import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction; import com.google.devtools.build.lib.analysis.test.InstrumentedFilesInfo; @@ -3191,7 +3190,7 @@ public void testCompileWithDisablingCompileJarIsPrivateApi() throws Exception { getConfiguredTarget("//foo:custom"); - assertContainsEvent("Rule in 'foo' cannot use private API"); + assertContainsEvent("got unexpected keyword argument: enable_compile_jar_action"); } @Test @@ -3226,7 +3225,7 @@ public void testCompileWithClasspathResourcesIsPrivateApi() throws Exception { getConfiguredTarget("//foo:custom"); - assertContainsEvent("Rule in 'foo' cannot use private API"); + assertContainsEvent("got unexpected keyword argument: classpath_resources"); } @Test @@ -3246,7 +3245,7 @@ public void testGetBuildInfoArtifactsIsPrivateApi() throws Exception { getConfiguredTarget("//foo:custom"); - assertContainsEvent("Rule in 'foo' cannot use private API"); + assertContainsEvent("no field or method 'get_build_info'"); } @Test @@ -3266,7 +3265,7 @@ public void testIsGoogleLegacyApiEnabledIsPrivateAPI() throws Exception { getConfiguredTarget("//foo:custom"); - assertContainsEvent("Rule in 'foo' cannot use private API"); + assertContainsEvent("no field or method '_google_legacy_api_enabled'"); } @Test @@ -3286,7 +3285,7 @@ public void testInstanceOfProviderIsPrivateApi() throws Exception { getConfiguredTarget("//foo:custom"); - assertContainsEvent("Rule in 'foo' cannot use private API"); + assertContainsEvent("no field or method 'check_provider_instances'"); } @Test @@ -3319,7 +3318,7 @@ public void testInjectingRuleKindIsPrivateApi() throws Exception { getConfiguredTarget("//foo:custom"); - assertContainsEvent("Rule in 'foo' cannot use private API"); + assertContainsEvent("got unexpected keyword argument: injecting_rule_kind"); } @Test @@ -3352,7 +3351,7 @@ public void testEnableJSpecifyIsPrivateApi() throws Exception { getConfiguredTarget("//foo:custom"); - assertContainsEvent("Rule in 'foo' cannot use private API"); + assertContainsEvent("got unexpected keyword argument: enable_jspecify"); } @Test @@ -3378,7 +3377,7 @@ public void testMergeJavaOutputsIsPrivateApi() throws Exception { getConfiguredTarget("//foo:custom"); - assertContainsEvent("Rule in 'foo' cannot use private API"); + assertContainsEvent("got unexpected keyword argument: merge_java_outputs"); } @Test @@ -3404,7 +3403,7 @@ public void testMergeSourceJarsIsPrivateApi() throws Exception { getConfiguredTarget("//foo:custom"); - assertContainsEvent("Rule in 'foo' cannot use private API"); + assertContainsEvent("got unexpected keyword argument: merge_source_jars"); } @Test @@ -3437,7 +3436,7 @@ public void testCompileIncludeCompilationInfoIsPrivateApi() throws Exception { getConfiguredTarget("//foo:custom"); - assertContainsEvent("Rule in 'foo' cannot use private API"); + assertContainsEvent("got unexpected keyword argument: include_compilation_info"); } @Test @@ -3495,7 +3494,7 @@ public void disallowJavaImportEmptyJars_fails() throws Exception { } @Test - public void testRunIjarIsPrivateApi() throws Exception { + public void testRunIjarWithOutputParameterIsPrivateApi() throws Exception { JavaToolchainTestUtil.writeBuildFileForJavaToolchain(scratch); scratch.file( "foo/custom_rule.bzl", @@ -3524,31 +3523,7 @@ public void testRunIjarIsPrivateApi() throws Exception { getConfiguredTarget("//foo:custom"); - assertContainsEvent("Rule in 'foo' cannot use private API"); - } - - @Test - public void testGetBuildInfoArtifacts() throws Exception { - scratch.file( - "bazel_internal/test/custom_rule.bzl", - "def _impl(ctx):", - " artifacts = java_common.get_build_info(ctx, False)", - " return [DefaultInfo(files = depset(artifacts))]", - "custom_rule = rule(", - " implementation = _impl,", - " attrs = {},", - ")"); - scratch.file( - "bazel_internal/test/BUILD", - "load(':custom_rule.bzl', 'custom_rule')", - "custom_rule(name = 'custom')"); - - NestedSet artifacts = - getConfiguredTarget("//bazel_internal/test:custom") - .getProvider(FileProvider.class) - .getFilesToBuild(); - - assertThat(prettyArtifactNames(artifacts)).containsExactly("build-info-redacted.properties"); + assertContainsEvent("got unexpected keyword argument: output"); } @Test @@ -3643,7 +3618,7 @@ public void testCollectNativeLibsDirsIsPrivateApi() throws Exception { getConfiguredTarget("//foo:custom"); - assertContainsEvent("Rule in 'foo' cannot use private API"); + assertContainsEvent("no field or method 'collect_native_deps_dirs'"); } @Test @@ -3664,7 +3639,7 @@ public void testGetRuntimeClasspathForArchiveIsPrivateApi() throws Exception { getConfiguredTarget("//foo:custom"); - assertContainsEvent("Rule in 'foo' cannot use private API"); + assertContainsEvent("no field or method 'get_runtime_classpath_for_archive'"); } @Test From 92d6f2245e4209a52d4d4e994a01d914b2c83df3 Mon Sep 17 00:00:00 2001 From: Googler Date: Tue, 4 Jul 2023 02:44:12 -0700 Subject: [PATCH 06/51] Automatic code cleanup. PiperOrigin-RevId: 545388423 Change-Id: Ic6e201d8dc36b8f453dac2695b32a8f9f3ea0064 --- .../devtools/coverageoutputgenerator/LcovMergerTestUtils.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/test/CoverageOutputGenerator/javatests/com/google/devtools/coverageoutputgenerator/LcovMergerTestUtils.java b/tools/test/CoverageOutputGenerator/javatests/com/google/devtools/coverageoutputgenerator/LcovMergerTestUtils.java index 64216d5d80b3db..316118b0cf1ff9 100644 --- a/tools/test/CoverageOutputGenerator/javatests/com/google/devtools/coverageoutputgenerator/LcovMergerTestUtils.java +++ b/tools/test/CoverageOutputGenerator/javatests/com/google/devtools/coverageoutputgenerator/LcovMergerTestUtils.java @@ -195,8 +195,6 @@ public class LcovMergerTestUtils { static final String SOURCE_FILENAME = "SOURCE_FILENAME"; static final int NR_FUNCTIONS_FOUND = 3; - static final int NR_FUNCTIONS_HIT_TRACEFILE1 = 2; - static final int NR_FUNCTIONS_HIT_TRACEFILE2 = 3; static final String FUNC_1 = "file1-func1"; static final int FUNC_1_LINE_NR = 10; @@ -214,8 +212,6 @@ public class LcovMergerTestUtils { static final long FUNC_3_NR_EXECUTED_LINES_TRACEFILE2 = 2; static final int NR_LINES_FOUND = 14; - static final int NR_LINES_HIT_TRACEFILE1 = 10; - static final int NR_LINES_HIT_TRACEFILE2 = 13; static final int NR_BRANCHES_FOUND = 6; static final int NR_BRANCHES_HIT_TRACEFILE1 = 2; From ef54ef5d17a013c863c4e2fb0583e6bd209645f2 Mon Sep 17 00:00:00 2001 From: Googler Date: Tue, 4 Jul 2023 02:59:09 -0700 Subject: [PATCH 07/51] Expose metadata_files parameter in coverage_common.instrumented_files_info Allows all rules to provide additional metadata files to the InstrumentedFilesInfo provider. It was always a little odd that the metadata_files attribute on the provider was exposed, but there was no way of populating it from Starlark. The other parameters are still considered too tied up with internal details at the moment to be exposed at this time. RELNOTES: coverage_common.instrumented_files_info now has a metadata_files argument PiperOrigin-RevId: 545391150 Change-Id: I3febdaf23c82d60c32b241a3935da27fd086fb7a --- .../devtools/build/lib/analysis/test/CoverageCommon.java | 7 +++---- .../build/lib/starlarkbuildapi/test/CoverageCommonApi.java | 4 +++- .../build/lib/starlark/StarlarkIntegrationTest.java | 5 ++++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/analysis/test/CoverageCommon.java b/src/main/java/com/google/devtools/build/lib/analysis/test/CoverageCommon.java index 62998c94d2404e..2c6b46a4279ffd 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/test/CoverageCommon.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/test/CoverageCommon.java @@ -100,10 +100,9 @@ public InstrumentedFilesInfoApi instrumentedFilesInfo( // Should have been verified by Starlark before this function is called throw new IllegalStateException(); } - if (!supportFilesBuilder.isEmpty() || !environmentPairs.isEmpty()) { - BuiltinRestriction.throwIfNotBuiltinUsage(thread); - } - if (!metadataFiles.isEmpty() || !reportedToActualSources.isEmpty()) { + if (!supportFilesBuilder.isEmpty() + || !reportedToActualSources.isEmpty() + || !environmentPairs.isEmpty()) { BuiltinRestriction.throwIfNotBuiltinUsage(thread); } return createInstrumentedFilesInfo( diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/test/CoverageCommonApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/test/CoverageCommonApi.java index 30a47151a598ca..bdf1f789386aa3 100644 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/test/CoverageCommonApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/test/CoverageCommonApi.java @@ -99,9 +99,11 @@ public interface CoverageCommonApi< defaultValue = "None"), @Param( name = "metadata_files", + doc = + "Additional files required to generate coverage LCOV files after code execution." + + " e.g. .gcno files for C++.", named = true, positional = false, - documented = false, defaultValue = "[]", allowedTypes = { @ParamType(type = Sequence.class, generic1 = FileApi.class), diff --git a/src/test/java/com/google/devtools/build/lib/starlark/StarlarkIntegrationTest.java b/src/test/java/com/google/devtools/build/lib/starlark/StarlarkIntegrationTest.java index ee5f77ff6760be..dffcdba8199da0 100644 --- a/src/test/java/com/google/devtools/build/lib/starlark/StarlarkIntegrationTest.java +++ b/src/test/java/com/google/devtools/build/lib/starlark/StarlarkIntegrationTest.java @@ -948,6 +948,8 @@ public void testInstrumentedFilesInfo_coverageEnabled() throws Exception { "load('//myinfo:myinfo.bzl', 'MyInfo')", "", "def custom_rule_impl(ctx):", + " metadata = ctx.actions.declare_file(ctx.label.name + '.metadata')", + " ctx.actions.write(metadata, '')", " return [", " coverage_common.instrumented_files_info(", " ctx,", @@ -967,6 +969,7 @@ public void testInstrumentedFilesInfo_coverageEnabled() throws Exception { // Missing attrs are ignored " 'missing_dep_attr',", " ],", + " metadata_files = [metadata],", " ),", " ]", "", @@ -1033,7 +1036,7 @@ public void testInstrumentedFilesInfo_coverageEnabled() throws Exception { assertThat( ActionsTestUtil.baseArtifactNames( ((Depset) myInfo.getValue("metadata_files")).getSet(Artifact.class))) - .containsExactly("label_dep.gcno", "label_list_dep.gcno", "dict_dep.gcno"); + .containsExactly("label_dep.gcno", "label_list_dep.gcno", "dict_dep.gcno", "cr.metadata"); ConfiguredTarget customRule = getConfiguredTarget("//test/starlark:cr"); assertThat( ActionsTestUtil.baseArtifactNames( From f551289281a2cbe20d898c4f4dcb22d4bc3211b0 Mon Sep 17 00:00:00 2001 From: Googler Date: Tue, 4 Jul 2023 04:17:37 -0700 Subject: [PATCH 08/51] Delete JavaBinary implementation and related unused code PiperOrigin-RevId: 545406467 Change-Id: Ib7b579b463f838827260096b645b3829cb7889fb --- .../lib/bazel/rules/java/BazelJavaBinary.java | 26 - .../bazel/rules/java/BazelJavaBinaryRule.java | 11 +- .../bazel/rules/java/BazelJavaSemantics.java | 214 ----- .../lib/bazel/rules/java/BazelJavaTest.java | 27 - .../bazel/rules/java/BazelJavaTestRule.java | 10 +- .../devtools/build/lib/rules/java/BUILD | 8 - .../build/lib/rules/java/JavaBinary.java | 797 ------------------ .../build/lib/rules/java/JavaCommon.java | 67 -- .../lib/rules/java/JavaCompilationHelper.java | 16 +- .../build/lib/rules/java/JavaSemantics.java | 66 -- .../lib/rules/java/JavaTargetAttributes.java | 9 - 11 files changed, 15 insertions(+), 1236 deletions(-) delete mode 100644 src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinary.java delete mode 100644 src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaTest.java delete mode 100644 src/main/java/com/google/devtools/build/lib/rules/java/JavaBinary.java diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinary.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinary.java deleted file mode 100644 index 849ebe9497cc98..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinary.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2014 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.bazel.rules.java; - -import com.google.devtools.build.lib.rules.java.JavaBinary; - -/** - * Implementation of {@code java_binary} with Bazel semantics. - */ -public class BazelJavaBinary extends JavaBinary { - public BazelJavaBinary() { - super(BazelJavaSemantics.INSTANCE); - } -} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinaryRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinaryRule.java index 87bd95df34ff60..9a735e785fc519 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinaryRule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinaryRule.java @@ -18,7 +18,8 @@ import static com.google.devtools.build.lib.packages.BuildType.LABEL_LIST; import static com.google.devtools.build.lib.packages.Type.BOOLEAN; -import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BaseRuleClasses.BinaryBaseRule; +import com.google.devtools.build.lib.analysis.BaseRuleClasses.EmptyRuleConfiguredTargetFactory; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; import com.google.devtools.build.lib.bazel.rules.java.BazelJavaRuleClasses.BaseJavaBinaryRule; @@ -32,6 +33,8 @@ /** * Rule definition for the java_binary rule. + * + *

This rule is implemented in Starlark. This class remains only for doc-gen purposes. */ public final class BazelJavaBinaryRule implements RuleDefinition { @Override @@ -107,10 +110,10 @@ public Object getDefault(AttributeMap rule) { @Override public Metadata getMetadata() { - return RuleDefinition.Metadata.builder() + return Metadata.builder() .name("java_binary") - .ancestors(BaseJavaBinaryRule.class, BaseRuleClasses.BinaryBaseRule.class) - .factoryClass(BazelJavaBinary.class) + .ancestors(BaseJavaBinaryRule.class, BinaryBaseRule.class) + .factoryClass(EmptyRuleConfiguredTargetFactory.class) .build(); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaSemantics.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaSemantics.java index 5f09384ecfb976..eb8c51e49a79cb 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaSemantics.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaSemantics.java @@ -14,9 +14,6 @@ package com.google.devtools.build.lib.bazel.rules.java; -import static com.google.common.base.Strings.isNullOrEmpty; - -import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -24,7 +21,6 @@ import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.analysis.RuleContext; import com.google.devtools.build.lib.analysis.Runfiles; -import com.google.devtools.build.lib.analysis.Runfiles.Builder; import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; import com.google.devtools.build.lib.analysis.actions.LauncherFileWriteAction; @@ -35,16 +31,12 @@ import com.google.devtools.build.lib.analysis.actions.Substitution.ComputedSubstitution; import com.google.devtools.build.lib.analysis.actions.Template; import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction; -import com.google.devtools.build.lib.analysis.test.TestConfiguration; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; import com.google.devtools.build.lib.collect.nestedset.Order; import com.google.devtools.build.lib.packages.BuildType; import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; -import com.google.devtools.build.lib.packages.Type; -import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; -import com.google.devtools.build.lib.rules.cpp.CcToolchainProvider; import com.google.devtools.build.lib.rules.java.DeployArchiveBuilder; import com.google.devtools.build.lib.rules.java.DeployArchiveBuilder.Compression; import com.google.devtools.build.lib.rules.java.JavaBuildInfoFactory; @@ -55,9 +47,7 @@ import com.google.devtools.build.lib.rules.java.JavaCompilationHelper; import com.google.devtools.build.lib.rules.java.JavaConfiguration; import com.google.devtools.build.lib.rules.java.JavaConfiguration.OneVersionEnforcementLevel; -import com.google.devtools.build.lib.rules.java.JavaHelper; import com.google.devtools.build.lib.rules.java.JavaRuleOutputJarsProvider; -import com.google.devtools.build.lib.rules.java.JavaRuntimeInfo; import com.google.devtools.build.lib.rules.java.JavaSemantics; import com.google.devtools.build.lib.rules.java.JavaSourceJarsProvider; import com.google.devtools.build.lib.rules.java.JavaTargetAttributes; @@ -68,9 +58,7 @@ import com.google.devtools.build.lib.shell.ShellUtils.TokenizationException; import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; import com.google.devtools.build.lib.util.OS; -import com.google.devtools.build.lib.util.Pair; import com.google.devtools.build.lib.util.ShellEscaper; -import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.PathFragment; import java.io.File; import java.util.ArrayList; @@ -117,75 +105,11 @@ public ImmutableList getBuildInfo(RuleContext ruleContext, int stamp) public void checkForProtoLibraryAndJavaProtoLibraryOnSameProto( RuleContext ruleContext, JavaCommon javaCommon) {} - private static final String JUNIT4_RUNNER = "org.junit.runner.JUnitCore"; - @Override public String getTestRunnerMainClass() { return BAZEL_TEST_RUNNER_MAIN_CLASS; } - @Nullable - private String getMainClassInternal(RuleContext ruleContext, ImmutableList sources) { - if (!ruleContext.attributes().get("create_executable", Type.BOOLEAN)) { - return null; - } - String mainClass = getMainClassFromRule(ruleContext); - - if (mainClass.isEmpty()) { - mainClass = JavaCommon.determinePrimaryClass(ruleContext, sources); - } - return mainClass; - } - - private String getMainClassFromRule(RuleContext ruleContext) { - String mainClass = ruleContext.attributes().get("main_class", Type.STRING); - if (!mainClass.isEmpty()) { - return mainClass; - } - - if (JavaSemantics.useLegacyJavaTest(ruleContext)) { - // Legacy behavior for java_test rules: main_class defaulted to JUnit4 runner. - // TODO(dmarting): remove once we drop the legacy bazel java_test behavior. - if ("java_test".equals(ruleContext.getRule().getRuleClass())) { - return JUNIT4_RUNNER; - } - } else { - if (ruleContext.attributes().get("use_testrunner", Type.BOOLEAN)) { - return BAZEL_TEST_RUNNER_MAIN_CLASS; - } - } - return mainClass; - } - - private void checkMainClass(RuleContext ruleContext, ImmutableList sources) { - boolean createExecutable = ruleContext.attributes().get("create_executable", Type.BOOLEAN); - String mainClass = getMainClassInternal(ruleContext, sources); - - if (!createExecutable && !isNullOrEmpty(mainClass)) { - ruleContext.ruleError("main class must not be specified when executable is not created"); - } - - if (createExecutable && isNullOrEmpty(mainClass)) { - if (sources.isEmpty()) { - ruleContext.ruleError("need at least one of 'main_class' or Java source files"); - } - mainClass = JavaCommon.determinePrimaryClass(ruleContext, sources); - if (mainClass == null) { - ruleContext.ruleError( - String.format( - "main_class was not provided and cannot be inferred: source path doesn't include" - + " a known root (%s)", - Joiner.on(", ").join(JavaUtil.KNOWN_SOURCE_ROOTS))); - } - } - } - - @Override - public String getMainClass(RuleContext ruleContext, ImmutableList sources) { - checkMainClass(ruleContext, sources); - return getMainClassInternal(ruleContext, sources); - } - @Override public ImmutableList collectResources(RuleContext ruleContext) { if (!ruleContext.getRule().isAttrDefined("resources", BuildType.LABEL_LIST)) { @@ -416,19 +340,6 @@ private static NestedSet getRuntimeJarsForTargets(TransitiveInfoCollec .getRuntimeJars(); } - @Override - public void addRunfilesForBinary( - RuleContext ruleContext, Artifact launcher, Runfiles.Builder runfilesBuilder) - throws RuleErrorException { - TransitiveInfoCollection testSupport = JavaSemantics.getTestSupport(ruleContext); - if (testSupport != null) { - // We assume that the runtime jars will not have conflicting artifacts - // with the same root relative path - runfilesBuilder.addTransitiveArtifactsWrappedInStableOrder( - getRuntimeJarsForTargets(testSupport)); - } - } - @Override public void addRunfilesForLibrary(RuleContext ruleContext, Runfiles.Builder runfilesBuilder) { } @@ -461,88 +372,6 @@ public ImmutableList getCompatibleJavacOptions( return ImmutableList.of(); } - // TODO(dmarting): simplify that logic when we remove the legacy Bazel java_test behavior. - @Nullable - private String getPrimaryClassLegacy(RuleContext ruleContext, ImmutableList sources) { - boolean createExecutable = ruleContext.attributes().get("create_executable", Type.BOOLEAN); - if (!createExecutable) { - return null; - } - return getMainClassInternal(ruleContext, sources); - } - - @Nullable - private String getPrimaryClassNew(RuleContext ruleContext, ImmutableList sources) { - boolean createExecutable = ruleContext.attributes().get("create_executable", Type.BOOLEAN); - - if (!createExecutable) { - return null; - } - - boolean useTestrunner = ruleContext.attributes().get("use_testrunner", Type.BOOLEAN); - - String testClass = ruleContext.getRule().isAttrDefined("test_class", Type.STRING) - ? ruleContext.attributes().get("test_class", Type.STRING) : ""; - - if (useTestrunner) { - if (testClass.isEmpty()) { - testClass = JavaCommon.determinePrimaryClass(ruleContext, sources); - if (testClass == null) { - ruleContext.ruleError("cannot determine junit.framework.Test class " - + "(Found no source file '" + ruleContext.getTarget().getName() - + ".java' and package name doesn't include 'java' or 'javatests'. " - + "You might want to rename the rule or add a 'test_class' " - + "attribute.)"); - } - } - return testClass; - } else { - if (!testClass.isEmpty()) { - ruleContext.attributeError("test_class", "this attribute is only meaningful to " - + "BazelTestRunner, but you are not using it (use_testrunner = 0)"); - } - return getMainClassInternal(ruleContext, sources); - } - } - - @Override - public String getPrimaryClass(RuleContext ruleContext, ImmutableList sources) { - return JavaSemantics.useLegacyJavaTest(ruleContext) - ? getPrimaryClassLegacy(ruleContext, sources) - : getPrimaryClassNew(ruleContext, sources); - } - - @Override - public Iterable getJvmFlags( - RuleContext ruleContext, ImmutableList sources, List userJvmFlags) { - ImmutableList.Builder jvmFlags = ImmutableList.builder(); - jvmFlags.addAll(userJvmFlags); - - if (!JavaSemantics.useLegacyJavaTest(ruleContext)) { - if (ruleContext.attributes().get("use_testrunner", Type.BOOLEAN)) { - String testClass = ruleContext.getRule().isAttrDefined("test_class", Type.STRING) - ? ruleContext.attributes().get("test_class", Type.STRING) : ""; - if (testClass.isEmpty()) { - testClass = JavaCommon.determinePrimaryClass(ruleContext, sources); - } - - if (testClass == null) { - ruleContext.ruleError("cannot determine test class"); - } else { - if (JavaRuntimeInfo.from(ruleContext).version() >= 17) { - jvmFlags.add("-Djava.security.manager=allow"); - } - // Always run junit tests with -ea (enable assertion) - jvmFlags.add("-ea"); - // "suite" is a misnomer. - jvmFlags.add("-Dbazel.test_suite=" + ShellEscaper.escapeString(testClass)); - } - } - } - - return jvmFlags.build(); - } - @Override public String addCoverageSupport(JavaCompilationHelper helper, Artifact executable) { // This method can be called only for *_binary/*_test targets. @@ -597,27 +426,6 @@ public CustomCommandLine buildSingleJarCommandLine( .build(); } - @Override - public ImmutableList translate(RuleContext ruleContext, List messages) { - return ImmutableList.of(); - } - - @Override - public Pair getLauncher( - RuleContext ruleContext, - JavaCommon common, - DeployArchiveBuilder deployArchiveBuilder, - DeployArchiveBuilder unstrippedDeployArchiveBuilder, - Builder runfilesBuilder, - List jvmFlags, - JavaTargetAttributes.Builder attributesBuilder, - boolean shouldStrip, - CcToolchainProvider ccToolchain, - FeatureConfiguration featureConfiguration) { - Artifact launcher = JavaHelper.launcherArtifactForTarget(this, ruleContext); - return new Pair<>(launcher, launcher); - } - @Override public void addArtifactToJavaTargetAttribute(JavaTargetAttributes.Builder builder, Artifact srcArtifact) { @@ -636,28 +444,6 @@ public PathFragment getDefaultJavaResourcePath(PathFragment path) { return javaPath == null ? path : javaPath; } - @Override - public List getExtraArguments(RuleContext ruleContext, ImmutableList sources) { - if (ruleContext.getRule().getRuleClass().equals("java_test")) { - if (JavaSemantics.useLegacyJavaTest(ruleContext)) { - TestConfiguration testConfiguration = - ruleContext.getConfiguration().getFragment(TestConfiguration.class); - if ((testConfiguration == null || testConfiguration.getTestArguments().isEmpty()) - && !ruleContext.attributes().isAttributeValueExplicitlySpecified("args")) { - ImmutableList.Builder builder = ImmutableList.builder(); - for (Artifact artifact : sources) { - PathFragment path = artifact.getRepositoryRelativePath(); - String className = JavaUtil.getJavaFullClassname(FileSystemUtils.removeExtension(path)); - if (className != null) { - builder.add(className); - } - } - return builder.build(); - } - } - } - return ImmutableList.of(); - } @Override public Artifact getProtoMapping(RuleContext ruleContext) throws InterruptedException { return null; diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaTest.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaTest.java deleted file mode 100644 index fb33cef811b0a0..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaTest.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2014 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.bazel.rules.java; - -import com.google.devtools.build.lib.analysis.RuleConfiguredTargetFactory; -import com.google.devtools.build.lib.rules.java.JavaBinary; - -/** - * An implementation of {@code java_test} rules. - */ -public class BazelJavaTest extends JavaBinary implements RuleConfiguredTargetFactory { - public BazelJavaTest() { - super(BazelJavaSemantics.INSTANCE); - } -} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaTestRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaTestRule.java index 35eb7943f27fec..2230e63b33cbb4 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaTestRule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaTestRule.java @@ -21,6 +21,8 @@ import static com.google.devtools.build.lib.packages.Type.STRING; import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BaseRuleClasses.EmptyRuleConfiguredTargetFactory; +import com.google.devtools.build.lib.analysis.BaseRuleClasses.TestBaseRule; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; import com.google.devtools.build.lib.analysis.config.ExecutionTransitionFactory; @@ -34,6 +36,8 @@ /** * Rule definition for the java_test rule. + * + *

This rule is implemented in Starlark. This class remains only for doc-gen purposes. */ public final class BazelJavaTestRule implements RuleDefinition { @@ -106,11 +110,11 @@ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) @Override public Metadata getMetadata() { - return RuleDefinition.Metadata.builder() + return Metadata.builder() .name("java_test") .type(RuleClassType.TEST) - .ancestors(BaseJavaBinaryRule.class, BaseRuleClasses.TestBaseRule.class) - .factoryClass(BazelJavaTest.class) + .ancestors(BaseJavaBinaryRule.class, TestBaseRule.class) + .factoryClass(EmptyRuleConfiguredTargetFactory.class) .build(); } } diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/BUILD b/src/main/java/com/google/devtools/build/lib/rules/java/BUILD index f30234ff10d3de..98f41cbc2fbf9e 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/BUILD +++ b/src/main/java/com/google/devtools/build/lib/rules/java/BUILD @@ -15,7 +15,6 @@ java_library( name = "java-rules", srcs = [ "GenericBuildInfoPropertiesTranslator.java", - "JavaBinary.java", "JavaImportBaseRule.java", "JavaInfo.java", "JavaPackageConfiguration.java", @@ -36,11 +35,9 @@ java_library( ":java-compilation", "//src/main/java/com/google/devtools/build/lib/actions", "//src/main/java/com/google/devtools/build/lib/actions:artifacts", - "//src/main/java/com/google/devtools/build/lib/actions:execution_requirements", "//src/main/java/com/google/devtools/build/lib/analysis:actions/custom_command_line", "//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster", "//src/main/java/com/google/devtools/build/lib/analysis:config/build_configuration", - "//src/main/java/com/google/devtools/build/lib/analysis:config/compilation_mode", "//src/main/java/com/google/devtools/build/lib/analysis:config/execution_transition_factory", "//src/main/java/com/google/devtools/build/lib/analysis:config/transitions/no_transition", "//src/main/java/com/google/devtools/build/lib/analysis:configured_target", @@ -48,7 +45,6 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:package_specification_provider", "//src/main/java/com/google/devtools/build/lib/analysis:rule_definition_environment", "//src/main/java/com/google/devtools/build/lib/analysis:template_variable_info", - "//src/main/java/com/google/devtools/build/lib/analysis:test/execution_info", "//src/main/java/com/google/devtools/build/lib/analysis:transitive_info_collection", "//src/main/java/com/google/devtools/build/lib/analysis:transitive_info_provider", "//src/main/java/com/google/devtools/build/lib/analysis/platform", @@ -58,7 +54,6 @@ java_library( "//src/main/java/com/google/devtools/build/lib/packages", "//src/main/java/com/google/devtools/build/lib/packages:exec_group", "//src/main/java/com/google/devtools/build/lib/packages/semantics", - "//src/main/java/com/google/devtools/build/lib/rules/apple", "//src/main/java/com/google/devtools/build/lib/rules/cpp", "//src/main/java/com/google/devtools/build/lib/starlarkbuildapi", "//src/main/java/com/google/devtools/build/lib/starlarkbuildapi/core", @@ -66,9 +61,6 @@ java_library( "//src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java", "//src/main/java/com/google/devtools/build/lib/util", "//src/main/java/com/google/devtools/build/lib/util:filetype", - "//src/main/java/com/google/devtools/build/lib/util:os", - "//src/main/java/com/google/devtools/build/lib/util:string", - "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//src/main/java/net/starlark/java/eval", "//src/main/java/net/starlark/java/syntax", diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaBinary.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaBinary.java deleted file mode 100644 index de564a1fa3df3c..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaBinary.java +++ /dev/null @@ -1,797 +0,0 @@ -// Copyright 2014 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.lib.rules.java; - -import static com.google.devtools.build.lib.collect.nestedset.Order.STABLE_ORDER; -import static com.google.devtools.build.lib.packages.ExecGroup.DEFAULT_EXEC_GROUP_NAME; -import static com.google.devtools.build.lib.packages.Type.BOOLEAN; -import static com.google.devtools.build.lib.rules.cpp.CppRuleClasses.JAVA_LAUNCHER_LINK; -import static com.google.devtools.build.lib.rules.cpp.CppRuleClasses.STATIC_LINKING_MODE; -import static com.google.devtools.build.lib.rules.java.DeployArchiveBuilder.Compression.COMPRESSED; -import static java.util.Objects.requireNonNull; - -import com.google.common.base.Predicates; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.actions.ExecutionRequirements; -import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException; -import com.google.devtools.build.lib.analysis.Allowlist; -import com.google.devtools.build.lib.analysis.ConfiguredTarget; -import com.google.devtools.build.lib.analysis.FilesToRunProvider; -import com.google.devtools.build.lib.analysis.OutputGroupInfo; -import com.google.devtools.build.lib.analysis.PrerequisiteArtifacts; -import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; -import com.google.devtools.build.lib.analysis.RuleConfiguredTargetFactory; -import com.google.devtools.build.lib.analysis.RuleContext; -import com.google.devtools.build.lib.analysis.Runfiles; -import com.google.devtools.build.lib.analysis.RunfilesProvider; -import com.google.devtools.build.lib.analysis.RunfilesSupport; -import com.google.devtools.build.lib.analysis.SourceManifestAction; -import com.google.devtools.build.lib.analysis.SourceManifestAction.ManifestType; -import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; -import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; -import com.google.devtools.build.lib.analysis.actions.FileWriteAction; -import com.google.devtools.build.lib.analysis.actions.SpawnAction; -import com.google.devtools.build.lib.analysis.config.CompilationMode; -import com.google.devtools.build.lib.analysis.test.ExecutionInfo; -import com.google.devtools.build.lib.collect.nestedset.NestedSet; -import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; -import com.google.devtools.build.lib.collect.nestedset.Order; -import com.google.devtools.build.lib.packages.BuildType; -import com.google.devtools.build.lib.packages.TargetUtils; -import com.google.devtools.build.lib.packages.Type; -import com.google.devtools.build.lib.rules.apple.ApplePlatform; -import com.google.devtools.build.lib.rules.cpp.CcCommon; -import com.google.devtools.build.lib.rules.cpp.CcCommon.Language; -import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; -import com.google.devtools.build.lib.rules.cpp.CcToolchainProvider; -import com.google.devtools.build.lib.rules.cpp.CppConfiguration; -import com.google.devtools.build.lib.rules.cpp.CppHelper; -import com.google.devtools.build.lib.rules.java.JavaCompilationArgsProvider.ClasspathType; -import com.google.devtools.build.lib.rules.java.JavaConfiguration.OneVersionEnforcementLevel; -import com.google.devtools.build.lib.rules.java.JavaRuleOutputJarsProvider.JavaOutput; -import com.google.devtools.build.lib.rules.java.proto.GeneratedExtensionRegistryProvider; -import com.google.devtools.build.lib.util.OS; -import com.google.devtools.build.lib.util.Pair; -import com.google.devtools.build.lib.util.StringCanonicalizer; -import com.google.devtools.build.lib.vfs.FileSystemUtils; -import com.google.devtools.build.lib.vfs.PathFragment; -import java.util.ArrayList; -import java.util.List; -import javax.annotation.Nullable; -import net.starlark.java.eval.EvalException; - -/** An implementation of java_binary. */ -public class JavaBinary implements RuleConfiguredTargetFactory { - private static final PathFragment CPP_RUNTIMES = PathFragment.create("_cpp_runtimes"); - - private final JavaSemantics semantics; - - protected JavaBinary(JavaSemantics semantics) { - this.semantics = semantics; - } - - @Override - @Nullable - public ConfiguredTarget create(RuleContext ruleContext) - throws InterruptedException, RuleErrorException, ActionConflictException { - final JavaCommon common = new JavaCommon(ruleContext, semantics); - DeployArchiveBuilder deployArchiveBuilder = new DeployArchiveBuilder(semantics, ruleContext); - Runfiles.Builder runfilesBuilder = - new Runfiles.Builder( - ruleContext.getWorkspaceName(), - ruleContext.getConfiguration().legacyExternalRunfiles()); - List jvmFlags = new ArrayList<>(); - - JavaTargetAttributes.Builder attributesBuilder = common.initCommon(); - attributesBuilder.addClassPathResources( - ruleContext.getPrerequisiteArtifacts("classpath_resources").list()); - - // Add Java8 timezone resource data - addTimezoneResourceForJavaBinaries(ruleContext, attributesBuilder); - - List userJvmFlags = JavaCommon.getJvmFlags(ruleContext); - - ruleContext.checkSrcsSamePackage(true); - boolean createExecutable = ruleContext.attributes().get("create_executable", Type.BOOLEAN); - - if (!createExecutable - && ruleContext.attributes().isAttributeValueExplicitlySpecified("launcher")) { - ruleContext.ruleError("launcher specified but create_executable is false"); - } - - if (!ruleContext.attributes().get("use_launcher", Type.BOOLEAN) - && ruleContext.attributes().isAttributeValueExplicitlySpecified("launcher")) { - ruleContext.ruleError("launcher specified but use_launcher is false"); - } - - if (ruleContext.attributes().isAttributeValueExplicitlySpecified("add_exports") - && Allowlist.hasAllowlist(ruleContext, "java_add_exports_allowlist") - && !Allowlist.isAvailable(ruleContext, "java_add_exports_allowlist")) { - ruleContext.ruleError("setting add_exports is not permitted"); - } - if (ruleContext.attributes().isAttributeValueExplicitlySpecified("add_opens") - && Allowlist.hasAllowlist(ruleContext, "java_add_opens_allowlist") - && !Allowlist.isAvailable(ruleContext, "java_add_opens_allowlist")) { - ruleContext.ruleError("setting add_opens is not permitted"); - } - - semantics.checkRule(ruleContext, common); - semantics.checkForProtoLibraryAndJavaProtoLibraryOnSameProto(ruleContext, common); - String mainClass = semantics.getMainClass(ruleContext, common.getSrcsArtifacts()); - String originalMainClass = mainClass; - if (ruleContext.hasErrors()) { - return null; - } - - // Collect the transitive dependencies. - JavaCompilationHelper helper = - new JavaCompilationHelper(ruleContext, semantics, common.getJavacOpts(), attributesBuilder); - List deps = - Lists.newArrayList(common.targetsTreatedAsDeps(ClasspathType.COMPILE_ONLY)); - helper.addLibrariesToAttributes(deps); - attributesBuilder.addNativeLibraries( - JavaCommon.collectNativeLibraries(common.targetsTreatedAsDeps(ClasspathType.BOTH))); - - // deploy_env is valid for java_binary, but not for java_test. - if (ruleContext.getRule().isAttrDefined("deploy_env", BuildType.LABEL_LIST)) { - for (JavaRuntimeClasspathProvider envTarget : - ruleContext.getPrerequisites("deploy_env", JavaRuntimeClasspathProvider.class)) { - attributesBuilder.addExcludedArtifacts(envTarget.getRuntimeClasspathNestedSet()); - } - } - - Artifact srcJar = ruleContext.getImplicitOutputArtifact(JavaSemantics.JAVA_BINARY_SOURCE_JAR); - JavaSourceJarsProvider.Builder javaSourceJarsProviderBuilder = - JavaSourceJarsProvider.builder() - .addSourceJar(srcJar) - .addAllTransitiveSourceJars(common.collectTransitiveSourceJars(srcJar)); - Artifact classJar = ruleContext.getImplicitOutputArtifact(JavaSemantics.JAVA_BINARY_CLASS_JAR); - - CppConfiguration cppConfiguration = - ruleContext.getConfiguration().getFragment(CppConfiguration.class); - CcToolchainProvider ccToolchain = - CppHelper.getToolchainUsingDefaultCcToolchainAttribute(ruleContext); - FeatureConfiguration featureConfiguration = null; - try { - featureConfiguration = - CcCommon.configureFeaturesOrThrowEvalException( - /* requestedFeatures= */ ImmutableSet.builder() - .addAll(ruleContext.getFeatures()) - .add(STATIC_LINKING_MODE) - .add(JAVA_LAUNCHER_LINK) - .build(), - /* unsupportedFeatures= */ ruleContext.getDisabledFeatures(), - Language.CPP, - ccToolchain, - cppConfiguration); - } catch (EvalException e) { - ruleContext.ruleError(e.getMessage()); - } - boolean stripAsDefault = - ccToolchain.shouldCreatePerObjectDebugInfo(featureConfiguration, cppConfiguration) - && cppConfiguration.getCompilationMode() == CompilationMode.OPT; - DeployArchiveBuilder unstrippedDeployArchiveBuilder = null; - if (stripAsDefault) { - unstrippedDeployArchiveBuilder = new DeployArchiveBuilder(semantics, ruleContext); - } - Pair launcherAndUnstrippedLauncher = - semantics.getLauncher( - ruleContext, - common, - deployArchiveBuilder, - unstrippedDeployArchiveBuilder, - runfilesBuilder, - jvmFlags, - attributesBuilder, - stripAsDefault, - ccToolchain, - featureConfiguration); - Artifact launcher = launcherAndUnstrippedLauncher.first; - Artifact unstrippedLauncher = launcherAndUnstrippedLauncher.second; - - JavaCompilationArtifacts.Builder javaArtifactsBuilder = new JavaCompilationArtifacts.Builder(); - - NestedSetBuilder filesBuilder = NestedSetBuilder.stableOrder(); - Artifact executableForRunfiles = null; - if (createExecutable) { - // This artifact is named as the rule itself, e.g. //foo:bar_bin -> bazel-bin/foo/bar_bin - // On Windows, it's going to be bazel-bin/foo/bar_bin.exe - if (OS.getCurrent() == OS.WINDOWS) { - executableForRunfiles = - ruleContext.getImplicitOutputArtifact(ruleContext.getTarget().getName() + ".exe"); - } else { - executableForRunfiles = ruleContext.createOutputArtifact(); - } - filesBuilder.add(classJar).add(executableForRunfiles); - - if (ruleContext.getConfiguration().isCodeCoverageEnabled()) { - mainClass = semantics.addCoverageSupport(helper, executableForRunfiles); - } - } else { - filesBuilder.add(classJar); - } - - JavaCompileOutputs outputs = helper.createOutputs(classJar); - JavaRuleOutputJarsProvider.Builder ruleOutputJarsProviderBuilder = - JavaRuleOutputJarsProvider.builder() - .addJavaOutput( - JavaOutput.builder().fromJavaCompileOutputs(outputs).addSourceJar(srcJar).build()); - - JavaTargetAttributes attributes = attributesBuilder.build(); - List nativeLibraries = attributes.getNativeLibraries(); - if (!nativeLibraries.isEmpty()) { - jvmFlags.add( - "-Djava.library.path=" - + JavaCommon.javaLibraryPath( - nativeLibraries, ruleContext.getRule().getPackage().getWorkspaceName())); - } - - JavaConfiguration javaConfig = ruleContext.getFragment(JavaConfiguration.class); - if (attributes.hasMessages()) { - helper.setTranslations(semantics.translate(ruleContext, attributes.getMessages())); - } - - if (attributes.hasSources() || attributes.hasResources()) { - // We only want to add a jar to the classpath of a dependent rule if it has content. - javaArtifactsBuilder.addRuntimeJar(classJar); - } - - GeneratedExtensionRegistryProvider generatedExtensionRegistryProvider = - semantics.createGeneratedExtensionRegistry( - ruleContext, - common, - filesBuilder, - javaArtifactsBuilder, - ruleOutputJarsProviderBuilder, - javaSourceJarsProviderBuilder); - javaArtifactsBuilder.setCompileTimeDependencies(outputs.depsProto()); - - JavaCompilationArtifacts javaArtifacts = javaArtifactsBuilder.build(); - common.setJavaCompilationArtifacts(javaArtifacts); - - helper.createCompileAction(outputs); - helper.createSourceJarAction(srcJar, outputs.genSource()); - - common.setClassPathFragment( - new ClasspathConfiguredFragment( - javaArtifacts, attributes, false, helper.getBootclasspathOrDefault())); - - Iterables.addAll( - jvmFlags, semantics.getJvmFlags(ruleContext, common.getSrcsArtifacts(), userJvmFlags)); - - JavaModuleFlagsProvider javaModuleFlagsProvider = - JavaModuleFlagsProvider.create( - ruleContext, - JavaInfo.moduleFlagsProviders(common.targetsTreatedAsDeps(ClasspathType.BOTH)) - .stream()); - - javaModuleFlagsProvider.toFlags().stream() - // Share strings in the heap with the equivalent javacopt flags, which are also interned - .map(StringCanonicalizer::intern) - .forEach(jvmFlags::add); - - if (ruleContext.hasErrors()) { - return null; - } - - Artifact executableToRun = executableForRunfiles; - if (createExecutable) { - String javaExecutable; - if (semantics.isJavaExecutableSubstitution()) { - javaExecutable = JavaCommon.getJavaBinSubstitution(ruleContext, launcher); - } else { - javaExecutable = JavaCommon.getJavaExecutableForStub(ruleContext, launcher); - } - // Create a shell stub for a Java application - executableToRun = - semantics.createStubAction( - ruleContext, - common, - jvmFlags, - executableForRunfiles, - mainClass, - originalMainClass, - filesBuilder, - javaExecutable, - /* createCoverageMetadataJar= */ false); - if (!executableToRun.equals(executableForRunfiles)) { - filesBuilder.add(executableToRun); - runfilesBuilder.addArtifact(executableToRun); - } - } - - JavaSourceJarsProvider sourceJarsProvider = javaSourceJarsProviderBuilder.build(); - NestedSet transitiveSourceJars = sourceJarsProvider.getTransitiveSourceJars(); - - // TODO(bazel-team): if (getOptions().sourceJars) then make this a dummy prerequisite for the - // DeployArchiveAction ? Needs a few changes there as we can't pass inputs - SingleJarActionBuilder.createSourceJarAction( - ruleContext, - semantics, - NestedSetBuilder.emptySet(Order.STABLE_ORDER), - transitiveSourceJars, - ruleContext.getImplicitOutputArtifact(JavaSemantics.JAVA_BINARY_DEPLOY_SOURCE_JAR), - ruleContext.useAutoExecGroups() - ? semantics.getJavaToolchainType() - : DEFAULT_EXEC_GROUP_NAME); - - RuleConfiguredTargetBuilder builder = new RuleConfiguredTargetBuilder(ruleContext); - builder.add( - JavaPrimaryClassProvider.class, - new JavaPrimaryClassProvider( - semantics.getPrimaryClass(ruleContext, common.getSrcsArtifacts()))); - if (generatedExtensionRegistryProvider != null) { - builder.addNativeDeclaredProvider(generatedExtensionRegistryProvider); - } - - Artifact deployJar = - ruleContext.getImplicitOutputArtifact(JavaSemantics.JAVA_BINARY_DEPLOY_JAR); - - if (javaConfig.oneVersionEnforcementLevel() != OneVersionEnforcementLevel.OFF) { - // This JavaBinary class is also the implementation for java_test targets (via the - // {Google,Bazel}JavaTest subclass). java_test targets can have their one version enforcement - // disabled with a second flag (to avoid the incremental build performance cost at the expense - // of safety.) - if (javaConfig.enforceOneVersionOnJavaTests() || !isJavaTestRule(ruleContext)) { - Artifact oneVersionOutputArtifact = - OneVersionCheckActionBuilder.newBuilder() - .withEnforcementLevel(javaConfig.oneVersionEnforcementLevel()) - .useToolchain(JavaToolchainProvider.from(ruleContext)) - .checkJars( - NestedSetBuilder.fromNestedSet(attributes.getRuntimeClassPath()) - .add(classJar) - .build()) - .build(ruleContext); - if (oneVersionOutputArtifact != null) { - builder.addOutputGroup(OutputGroupInfo.VALIDATION, oneVersionOutputArtifact); - } - } - } - NestedSet filesToBuild = filesBuilder.build(); - - NestedSet dynamicRuntimeActionInputs; - try { - dynamicRuntimeActionInputs = ccToolchain.getDynamicRuntimeLinkInputs(featureConfiguration); - } catch (EvalException e) { - throw ruleContext.throwWithRuleError(e); - } - - collectDefaultRunfiles( - runfilesBuilder, - ruleContext, - common, - javaArtifacts, - filesToBuild, - launcher, - dynamicRuntimeActionInputs); - Runfiles defaultRunfiles = runfilesBuilder.build(); - - RunfilesSupport runfilesSupport = null; - NestedSetBuilder extraFilesToRunBuilder = NestedSetBuilder.stableOrder(); - if (createExecutable) { - List extraArgs = - new ArrayList<>(semantics.getExtraArguments(ruleContext, common.getSrcsArtifacts())); - // The executable we pass here will be used when creating the runfiles directory. E.g. for the - // stub script called bazel-bin/foo/bar_bin, the runfiles directory will be created under - // bazel-bin/foo/bar_bin.runfiles . On platforms where there's an extra stub script (Windows) - // which dispatches to this one, we still create the runfiles directory for the shell script, - // but use the dispatcher script (a batch file) as the RunfilesProvider's executable. - runfilesSupport = - RunfilesSupport.withExecutable( - ruleContext, defaultRunfiles, executableForRunfiles, extraArgs); - extraFilesToRunBuilder.add(runfilesSupport.getRunfilesMiddleman()); - } - - RunfilesProvider runfilesProvider = - RunfilesProvider.withData( - defaultRunfiles, - new Runfiles.Builder( - ruleContext.getWorkspaceName(), - ruleContext.getConfiguration().legacyExternalRunfiles()) - .merge(runfilesSupport) - .build()); - - ImmutableList deployManifestLines = - getDeployManifestLines(ruleContext, originalMainClass); - - // Create the java_binary target specific CDS archive. - Artifact jsa = createSharedArchive(ruleContext, javaArtifacts, attributes); - - if (ruleContext.isAttrDefined("hermetic", BOOLEAN) - && ruleContext.attributes().get("hermetic", BOOLEAN)) { - if (!createExecutable) { - ruleContext.ruleError("hermetic specified but create_executable is false"); - } - - JavaRuntimeInfo javaRuntime = JavaRuntimeInfo.from(ruleContext); - if (!javaRuntime.hermeticInputs().isEmpty() - && javaRuntime.libModules() != null - && !javaRuntime.hermeticStaticLibs().isEmpty()) { - deployArchiveBuilder - .setJavaHome(javaRuntime.javaHomePathFragment()) - .setLibModules(javaRuntime.libModules()) - .setHermeticInputs(javaRuntime.hermeticInputs()); - } - - if (jsa == null) { - // Use the JDK default CDS specified by the JavaRuntime if the - // java_binary target specific CDS archive is null, when building - // a hermetic deploy JAR. - jsa = javaRuntime.defaultCDS(); - } - } - - deployArchiveBuilder - .setOutputJar(deployJar) - .setJavaStartClass(mainClass) - .setDeployManifestLines(deployManifestLines) - .setAttributes(attributes) - .addRuntimeJars(javaArtifacts.getRuntimeJars()) - .setIncludeBuildData(true) - .setRunfilesMiddleman( - runfilesSupport == null ? null : runfilesSupport.getRunfilesMiddleman()) - .setCompression(COMPRESSED) - .setLauncher(launcher) - .setOneVersionEnforcementLevel( - javaConfig.oneVersionEnforcementLevel(), - JavaToolchainProvider.from(ruleContext).getOneVersionAllowlist()) - .setMultiReleaseDeployJars(javaConfig.multiReleaseDeployJars()) - .setSharedArchive(jsa) - .setAddExports(javaModuleFlagsProvider.addExports()) - .setAddOpens(javaModuleFlagsProvider.addOpens()) - .build(); - - Artifact unstrippedDeployJar = - ruleContext.getImplicitOutputArtifact(JavaSemantics.JAVA_UNSTRIPPED_BINARY_DEPLOY_JAR); - if (stripAsDefault) { - requireNonNull(unstrippedDeployArchiveBuilder); // guarded by stripAsDefault - unstrippedDeployArchiveBuilder - .setOutputJar(unstrippedDeployJar) - .setJavaStartClass(mainClass) - .setDeployManifestLines(deployManifestLines) - .setAttributes(attributes) - .addRuntimeJars(javaArtifacts.getRuntimeJars()) - .setIncludeBuildData(true) - .setRunfilesMiddleman( - runfilesSupport == null ? null : runfilesSupport.getRunfilesMiddleman()) - .setCompression(COMPRESSED) - .setLauncher(unstrippedLauncher); - - unstrippedDeployArchiveBuilder.build(); - } else { - // Write an empty file as the name_deploy.jar.unstripped when the default output jar is not - // stripped. - ruleContext.registerAction( - FileWriteAction.create(ruleContext, unstrippedDeployJar, "", false)); - } - - JavaRuleOutputJarsProvider ruleOutputJarsProvider = ruleOutputJarsProviderBuilder.build(); - - JavaInfo.Builder javaInfoBuilder = JavaInfo.Builder.create(); - - NestedSetBuilder> coverageEnvironment = NestedSetBuilder.stableOrder(); - NestedSetBuilder coverageSupportFiles = NestedSetBuilder.stableOrder(); - if (ruleContext.getConfiguration().isCodeCoverageEnabled()) { - - // Create an artifact that contains the runfiles relative paths of the jars on the runtime - // classpath. Using SourceManifestAction is the only reliable way to match the runfiles - // creation code. - Artifact runtimeClasspathArtifact = - ruleContext.getUniqueDirectoryArtifact( - "runtime_classpath_for_coverage", - "runtime_classpath.txt", - ruleContext.getBinOrGenfilesDirectory()); - ruleContext.registerAction( - new SourceManifestAction( - ManifestType.SOURCES_ONLY, - ruleContext.getActionOwner(), - runtimeClasspathArtifact, - new Runfiles.Builder( - ruleContext.getWorkspaceName(), - ruleContext.getConfiguration().legacyExternalRunfiles()) - // This matches the code below in collectDefaultRunfiles. - .addTransitiveArtifactsWrappedInStableOrder(common.getRuntimeClasspath()) - .build(), - null, - true)); - filesBuilder.add(runtimeClasspathArtifact); - - // Pass the artifact through an environment variable in the coverage environment so it - // can be read by the coverage collection script. - coverageEnvironment.add( - new Pair<>( - "JAVA_RUNTIME_CLASSPATH_FOR_COVERAGE", runtimeClasspathArtifact.getExecPathString())); - // Add the file to coverageSupportFiles so it ends up as an input for the test action - // when coverage is enabled. - coverageSupportFiles.add(runtimeClasspathArtifact); - - // Make single jar reachable from the coverage environment because it needs to be executed - // by the coverage collection script. - FilesToRunProvider singleJar = JavaToolchainProvider.from(ruleContext).getSingleJar(); - coverageEnvironment.add( - new Pair<>("SINGLE_JAR_TOOL", singleJar.getExecutable().getExecPathString())); - coverageSupportFiles.addTransitive(singleJar.getFilesToRun()); - } - - common.addTransitiveInfoProviders( - builder, - javaInfoBuilder, - filesToBuild, - classJar, - coverageEnvironment.build(), - coverageSupportFiles.build()); - common.addGenJarsProvider(builder, javaInfoBuilder, outputs.genClass(), outputs.genSource()); - - // This rule specifically _won't_ build deploy_env, so we selectively propagate validations to - // filter out deploy_env's if there are any (and otherwise rely on automatic validation - // propagation). Note that any validations not propagated here will still be built if and when - // deploy_env is built. - if (ruleContext.getRule().isAttrDefined("deploy_env", BuildType.LABEL_LIST)) { - NestedSetBuilder excluded = NestedSetBuilder.stableOrder(); - for (OutputGroupInfo outputGroup : - ruleContext.getPrerequisites("deploy_env", OutputGroupInfo.STARLARK_CONSTRUCTOR)) { - NestedSet toExclude = outputGroup.getOutputGroup(OutputGroupInfo.VALIDATION); - if (!toExclude.isEmpty()) { - excluded.addTransitive(toExclude); - } - } - if (!excluded.isEmpty()) { - NestedSetBuilder validations = NestedSetBuilder.stableOrder(); - RuleConfiguredTargetBuilder.collectTransitiveValidationOutputGroups( - ruleContext, - attributeName -> !"deploy_env".equals(attributeName), - validations::addTransitive); - - // Likely, deploy_env will overlap with deps/runtime_deps. Unless we're building an - // executable (which is rare and somewhat questionable when deploy_env is specified), we can - // exclude validations from deploy_env entirely from this rule, since this rule specifically - // never builds the referenced code. - if (createExecutable) { - // Executable classpath isn't affected by deploy_env, so build all collected validations. - builder.addOutputGroup(OutputGroupInfo.VALIDATION_TRANSITIVE, validations.build()); - } else { - // Filter validations similar to JavaTargetAttributes.getRuntimeClassPathForArchive(). - builder.addOutputGroup( - OutputGroupInfo.VALIDATION_TRANSITIVE, - NestedSetBuilder.wrap( - Order.STABLE_ORDER, - Iterables.filter( - validations.build().toList(), - Predicates.not(Predicates.in(excluded.build().toSet()))))); - } - } - } - - Artifact validation = - AndroidLintActionBuilder.create( - ruleContext, - javaConfig, - attributes, - helper.getBootclasspathOrDefault(), - common, - semantics, - outputs); - if (validation != null) { - builder.addOutputGroup( - OutputGroupInfo.VALIDATION, NestedSetBuilder.create(STABLE_ORDER, validation)); - } - - // Support test execution on darwin. - if (ApplePlatform.isApplePlatform(ruleContext.getConfiguration().getCpu()) - && TargetUtils.isTestRule(ruleContext.getRule())) { - builder.addNativeDeclaredProvider( - new ExecutionInfo(ImmutableMap.of(ExecutionRequirements.REQUIRES_DARWIN, ""))); - } - - JavaInfo javaInfo = - javaInfoBuilder - .javaSourceJars(sourceJarsProvider) - .javaRuleOutputs(ruleOutputJarsProvider) - .build(); - - return builder - .setFilesToBuild(filesToBuild) - .addStarlarkDeclaredProvider(javaInfo) - .add(RunfilesProvider.class, runfilesProvider) - // The executable to run (below) may be different from the executable for runfiles (the one - // we create the runfiles support object with). On Linux they are the same (it's the same - // shell script), on Windows they are different (the executable to run is a batch file, the - // executable for runfiles is the shell script). - .setRunfilesSupport(runfilesSupport, executableToRun) - // Add the native libraries as test action tools. Useful for the persistent test runner - // to include them in the worker's key and re-build a worker if the native dependencies - // have changed. - .addTestActionTools(nativeLibraries) - .addFilesToRun(extraFilesToRunBuilder.build()) - .add( - JavaRuntimeClasspathProvider.class, - new JavaRuntimeClasspathProvider(common.getRuntimeClasspath())) - .addOutputGroup(JavaSemantics.SOURCE_JARS_OUTPUT_GROUP, transitiveSourceJars) - .addOutputGroup( - JavaSemantics.DIRECT_SOURCE_JARS_OUTPUT_GROUP, - NestedSetBuilder.wrap(Order.STABLE_ORDER, sourceJarsProvider.getSourceJars())) - .build(); - } - - @Nullable - private static Artifact createSharedArchive( - RuleContext ruleContext, - JavaCompilationArtifacts javaArtifacts, - JavaTargetAttributes attributes) - throws InterruptedException { - if (!ruleContext.getRule().isAttrDefined("classlist", BuildType.LABEL)) { - return null; - } - Artifact classlist = ruleContext.getPrerequisiteArtifact("classlist"); - if (classlist == null) { - return null; - } - NestedSet classpath = - NestedSetBuilder.stableOrder() - .addAll(javaArtifacts.getRuntimeJars()) - .addTransitive(attributes.getRuntimeClassPathForArchive()) - .build(); - Artifact jsa = ruleContext.getImplicitOutputArtifact(JavaSemantics.SHARED_ARCHIVE_ARTIFACT); - Artifact merged = - ruleContext.getDerivedArtifact( - jsa.getOutputDirRelativePath(ruleContext.getConfiguration().isSiblingRepositoryLayout()) - .replaceName( - FileSystemUtils.removeExtension(jsa.getRootRelativePath().getBaseName()) - + "-merged.jar"), - jsa.getRoot()); - SingleJarActionBuilder.createSingleJarAction(ruleContext, classpath, merged); - JavaRuntimeInfo javaRuntime = JavaRuntimeInfo.from(ruleContext); - Artifact configFile = ruleContext.getPrerequisiteArtifact("cds_config_file"); - - CustomCommandLine.Builder commandLine = - CustomCommandLine.builder() - .add("-Xshare:dump") - .addFormatted("-XX:SharedArchiveFile=%s", jsa.getExecPath()) - .addFormatted("-XX:SharedClassListFile=%s", classlist.getExecPath()); - if (configFile != null) { - commandLine.addFormatted("-XX:SharedArchiveConfigFile=%s", configFile.getExecPath()); - } - commandLine.add("-cp").addExecPath(merged); - SpawnAction.Builder spawnAction = new SpawnAction.Builder(); - if (ruleContext.getRule().isAttrDefined("jvm_flags_for_cds_image_creation", Type.STRING_LIST)) { - commandLine.addAll( - ruleContext - .getExpander() - .withDataExecLocations() - .list("jvm_flags_for_cds_image_creation")); - spawnAction.addTransitiveInputs(PrerequisiteArtifacts.nestedSet(ruleContext, "data")); - } - spawnAction - .setExecutable(javaRuntime.javaBinaryExecPathFragment()) - .addCommandLine(commandLine.build()) - .setMnemonic("JavaJSA") - .setProgressMessage("Dumping Java Shared Archive %s", jsa.prettyPrint()) - .addOutput(jsa) - .addInput(classlist) - .addInput(merged) - .addTransitiveInputs(javaRuntime.javaBaseInputs()); - if (configFile != null) { - spawnAction.addInput(configFile); - } - ruleContext.registerAction(spawnAction.build(ruleContext)); - return jsa; - } - - // Create the deploy jar and make it dependent on the runfiles middleman if an executable is - // created. Do not add the deploy jar to files to build, so we will only build it when it gets - // requested. - private static ImmutableList getDeployManifestLines( - RuleContext ruleContext, String originalMainClass) { - ImmutableList.Builder builder = - ImmutableList.builder() - .addAll(ruleContext.attributes().get("deploy_manifest_lines", Type.STRING_LIST)); - if (ruleContext.getConfiguration().isCodeCoverageEnabled()) { - builder.add("Coverage-Main-Class: " + originalMainClass); - } - return builder.build(); - } - - /** Add Java8 timezone resource jar to java binary, if specified in tool chain. */ - private static void addTimezoneResourceForJavaBinaries( - RuleContext ruleContext, JavaTargetAttributes.Builder attributesBuilder) { - JavaToolchainProvider toolchainProvider = JavaToolchainProvider.from(ruleContext); - if (toolchainProvider.getTimezoneData() != null) { - attributesBuilder.addResourceJars( - NestedSetBuilder.create(Order.STABLE_ORDER, toolchainProvider.getTimezoneData())); - } - } - - private void collectDefaultRunfiles( - Runfiles.Builder builder, - RuleContext ruleContext, - JavaCommon common, - JavaCompilationArtifacts javaArtifacts, - NestedSet filesToBuild, - Artifact launcher, - NestedSet dynamicRuntimeActionInputs) - throws RuleErrorException { - builder.addTransitiveArtifactsWrappedInStableOrder(filesToBuild); - builder.addArtifacts(javaArtifacts.getRuntimeJars()); - if (launcher != null) { - final TransitiveInfoCollection defaultLauncher = - JavaHelper.launcherForTarget(semantics, ruleContext); - final Artifact defaultLauncherArtifact = - JavaHelper.launcherArtifactForTarget(semantics, ruleContext); - if (!defaultLauncherArtifact.equals(launcher)) { - builder.addArtifact(launcher); - - // N.B. The "default launcher" referred to here is the launcher target specified through - // an attribute or flag. We wish to retain the runfiles of the default launcher, *except* - // for the original cc_binary artifact, because we've swapped it out with our custom - // launcher. Hence, instead of calling builder.addTarget(), or adding an odd method - // to Runfiles.Builder, we "unravel" the call and manually add things to the builder. - // Because the NestedSet representing each target's launcher runfiles is re-built here, - // we may see increased memory consumption for representing the target's runfiles. - Runfiles runfiles = - defaultLauncher.getProvider(RunfilesProvider.class).getDefaultRunfiles(); - NestedSetBuilder unconditionalArtifacts = NestedSetBuilder.compileOrder(); - for (Artifact a : runfiles.getArtifacts().toList()) { - if (!a.equals(defaultLauncherArtifact)) { - unconditionalArtifacts.add(a); - } - } - builder.addTransitiveArtifacts(unconditionalArtifacts.build()); - builder.addSymlinks(runfiles.getSymlinks()); - builder.addRootSymlinks(runfiles.getRootSymlinks()); - } else { - builder.addTarget( - defaultLauncher, - RunfilesProvider.DEFAULT_RUNFILES, - ruleContext.getConfiguration().alwaysIncludeFilesToBuildInData()); - } - } - - semantics.addRunfilesForBinary(ruleContext, launcher, builder); - builder.addRunfiles(ruleContext, RunfilesProvider.DEFAULT_RUNFILES); - - List runtimeDeps = - ruleContext.getPrerequisites("runtime_deps"); - builder.addTargets( - runtimeDeps, - RunfilesProvider.DEFAULT_RUNFILES, - ruleContext.getConfiguration().alwaysIncludeFilesToBuildInData()); - - builder.addTransitiveArtifactsWrappedInStableOrder(common.getRuntimeClasspath()); - - // Add the JDK files if it comes from the source repository (see java_stub_template.txt). - JavaRuntimeInfo javaRuntime = JavaRuntimeInfo.from(ruleContext); - if (javaRuntime != null) { - builder.addTransitiveArtifacts(javaRuntime.javaBaseInputs()); - - if (!javaRuntime.javaHomePathFragment().isAbsolute()) { - // Add symlinks to the C++ runtime libraries under a path that can be built - // into the Java binary without having to embed the crosstool, gcc, and grte - // version information contained within the libraries' package paths. - for (Artifact lib : dynamicRuntimeActionInputs.toList()) { - PathFragment path = CPP_RUNTIMES.getRelative(lib.getExecPath().getBaseName()); - builder.addSymlink(path, lib); - } - } - } - } - - private static boolean isJavaTestRule(RuleContext ruleContext) { - return ruleContext.getRule().getRuleClass().endsWith("_test"); - } -} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCommon.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCommon.java index 2147c6030ec46f..5b40d3f2081487 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCommon.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCommon.java @@ -14,7 +14,6 @@ package com.google.devtools.build.lib.rules.java; import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.google.devtools.build.lib.packages.Type.BOOLEAN; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -52,9 +51,7 @@ import com.google.devtools.build.lib.vfs.PathFragment; import java.util.ArrayList; import java.util.Collection; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Stream; import javax.annotation.Nullable; @@ -82,27 +79,6 @@ public class JavaCommon { private final JavaToolchainProvider javaToolchain; private JavaCompilationHelper javaCompilationHelper; - public JavaCommon(RuleContext ruleContext, JavaSemantics semantics) { - this( - ruleContext, - semantics, - ruleContext.getPrerequisiteArtifacts("srcs").list(), - collectTargetsTreatedAsDeps(ruleContext, semantics, ClasspathType.COMPILE_ONLY), - collectTargetsTreatedAsDeps(ruleContext, semantics, ClasspathType.RUNTIME_ONLY), - collectTargetsTreatedAsDeps(ruleContext, semantics, ClasspathType.BOTH)); - } - - public JavaCommon( - RuleContext ruleContext, JavaSemantics semantics, ImmutableList sources) { - this( - ruleContext, - semantics, - sources, - collectTargetsTreatedAsDeps(ruleContext, semantics, ClasspathType.COMPILE_ONLY), - collectTargetsTreatedAsDeps(ruleContext, semantics, ClasspathType.RUNTIME_ONLY), - collectTargetsTreatedAsDeps(ruleContext, semantics, ClasspathType.BOTH)); - } - public JavaCommon( RuleContext ruleContext, JavaSemantics semantics, @@ -179,33 +155,6 @@ public JavaCompilationArtifacts getJavaCompilationArtifacts() { return javaArtifacts; } - /** - * Creates the java.library.path from a list of the native libraries. Concatenates the parent - * directories of the shared libraries into a Java search path. Each relative path entry is - * prepended with "${JAVA_RUNFILES}/" so it can be resolved at runtime. - * - * @param sharedLibraries a collection of native libraries to create the java library path from - * @return a String containing the ":" separated java library path - */ - public static String javaLibraryPath(Collection sharedLibraries, String runfilePrefix) { - StringBuilder buffer = new StringBuilder(); - Set entries = new HashSet<>(); - for (Artifact sharedLibrary : sharedLibraries) { - PathFragment entry = sharedLibrary.getRootRelativePath().getParentDirectory(); - if (entries.add(entry)) { - if (buffer.length() > 0) { - buffer.append(':'); - } - buffer - .append("${JAVA_RUNFILES}/") - .append(runfilePrefix) - .append("/") - .append(entry.getPathString()); - } - } - return buffer.toString(); - } - /** * Collects Java compilation arguments for this target. * @@ -499,10 +448,6 @@ private static List getRuntimeDeps(RuleContext ruleCon } } - public JavaTargetAttributes.Builder initCommon() throws RuleErrorException, InterruptedException { - return initCommon(ImmutableList.of(), getCompatibleJavacOptions()); - } - /** * Initialize the common actions and build various collections of artifacts for the * initializationHook() methods of the subclasses. @@ -556,10 +501,6 @@ public JavaTargetAttributes.Builder initCommon( return javaTargetAttributes; } - private ImmutableList getCompatibleJavacOptions() { - return semantics.getCompatibleJavacOptions(ruleContext, javaToolchain); - } - private boolean disallowDepsWithoutSrcs(String ruleClass) { return ruleClass.equals("java_library") || ruleClass.equals("java_binary") @@ -571,14 +512,6 @@ public ImmutableList targetsTreatedAsDeps( return targetsTreatedAsDeps.get(type); } - public ImmutableList hermeticStaticLibs() { - if (ruleContext.isAttrDefined("hermetic", BOOLEAN) - && ruleContext.attributes().get("hermetic", BOOLEAN)) { - return JavaRuntimeInfo.from(ruleContext).hermeticStaticLibs(); - } - return ImmutableList.of(); - } - /** Returns the default dependencies for the given classpath context. */ public static ImmutableList defaultDeps( RuleContext ruleContext, JavaSemantics semantics, ClasspathType type) { diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationHelper.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationHelper.java index cc94e47b92b79a..c3a357d4f43aad 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationHelper.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationHelper.java @@ -74,8 +74,6 @@ public final class JavaCompilationHelper { private final JavaTargetAttributes.Builder attributes; private JavaTargetAttributes builtAttributes; private final ImmutableList customJavacOpts; - private final List translations = new ArrayList<>(); - private boolean translationsFrozen; private final JavaSemantics semantics; private final ImmutableList additionalInputsForDatabinding; private final StrictDepsMode strictJavaDeps; @@ -410,8 +408,7 @@ private boolean separateResourceJar( return !resourceJars.isEmpty() || !attributes.getResources().isEmpty() || !attributes.getResourceJars().isEmpty() - || !attributes.getClassPathResources().isEmpty() - || !getTranslations().isEmpty(); + || !attributes.getClassPathResources().isEmpty(); } private ImmutableMap getExecutionInfo() { @@ -658,7 +655,6 @@ private void createResourceJarAction(Artifact resourceJar, ImmutableList getJavacOpts() { return customJavacOpts; } - public void setTranslations(Collection translations) { - Preconditions.checkArgument(!translationsFrozen); - this.translations.addAll(translations); - } - - private ImmutableList getTranslations() { - translationsFrozen = true; - return ImmutableList.copyOf(translations); - } - /** * Creates the Action that creates ijars from Jar files. * diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaSemantics.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaSemantics.java index d9ecabeaefe2b2..60f4210433e318 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaSemantics.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaSemantics.java @@ -25,7 +25,6 @@ import com.google.devtools.build.lib.analysis.RuleContext; import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; import com.google.devtools.build.lib.analysis.Runfiles; -import com.google.devtools.build.lib.analysis.Runfiles.Builder; import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; import com.google.devtools.build.lib.analysis.actions.SpawnAction; @@ -37,15 +36,12 @@ import com.google.devtools.build.lib.packages.ImplicitOutputsFunction.SafeImplicitOutputsFunction; import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; import com.google.devtools.build.lib.packages.Type; -import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; -import com.google.devtools.build.lib.rules.cpp.CcToolchainProvider; import com.google.devtools.build.lib.rules.java.DeployArchiveBuilder.Compression; import com.google.devtools.build.lib.rules.java.JavaCompilationArgsProvider.ClasspathType; import com.google.devtools.build.lib.rules.java.JavaConfiguration.OneVersionEnforcementLevel; import com.google.devtools.build.lib.rules.java.proto.GeneratedExtensionRegistryProvider; import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; import com.google.devtools.build.lib.util.FileType; -import com.google.devtools.build.lib.util.Pair; import com.google.devtools.build.lib.vfs.PathFragment; import java.util.List; import javax.annotation.Nullable; @@ -59,7 +55,6 @@ public interface JavaSemantics { SafeImplicitOutputsFunction JAVA_BINARY_SOURCE_JAR = fromTemplates("%{name}-src.jar"); SafeImplicitOutputsFunction JAVA_BINARY_DEPLOY_JAR = fromTemplates("%{name}_deploy.jar"); - SafeImplicitOutputsFunction JAVA_BINARY_MERGED_JAR = fromTemplates("%{name}_merged.jar"); SafeImplicitOutputsFunction JAVA_UNSTRIPPED_BINARY_DEPLOY_JAR = fromTemplates("%{name}_deploy.jar.unstripped"); SafeImplicitOutputsFunction JAVA_BINARY_PROGUARD_MAP = fromTemplates("%{name}_proguard.map"); @@ -70,16 +65,10 @@ public interface JavaSemantics { SafeImplicitOutputsFunction JAVA_BINARY_PROGUARD_CONFIG = fromTemplates("%{name}_proguard.config"); SafeImplicitOutputsFunction JAVA_ONE_VERSION_ARTIFACT = fromTemplates("%{name}-one-version.txt"); - SafeImplicitOutputsFunction SHARED_ARCHIVE_ARTIFACT = fromTemplates("%{name}.jsa"); - - SafeImplicitOutputsFunction JAVA_COVERAGE_RUNTIME_CLASS_PATH_TXT = - fromTemplates("%{name}-runtime-classpath.txt"); SafeImplicitOutputsFunction JAVA_BINARY_DEPLOY_SOURCE_JAR = fromTemplates("%{name}_deploy-src.jar"); - SafeImplicitOutputsFunction JAVA_TEST_CLASSPATHS_FILE = fromTemplates("%{name}_classpaths_file"); - FileType JAVA_SOURCE = FileType.of(".java"); FileType JAR = FileType.of(".jar"); FileType PROPERTIES = FileType.of(".properties"); @@ -199,15 +188,6 @@ static LabelLateBoundDefault javaLauncherAttribute(Label defa void checkForProtoLibraryAndJavaProtoLibraryOnSameProto( RuleContext ruleContext, JavaCommon javaCommon); - /** Returns the main class of a Java binary. */ - String getMainClass(RuleContext ruleContext, ImmutableList srcsArtifacts); - - /** - * Returns the primary class for a Java binary - either the main class, or, in case of a test, the - * test class (not the test runner main class). - */ - String getPrimaryClass(RuleContext ruleContext, ImmutableList srcsArtifacts); - /** * Returns the resources contributed by a Java rule (usually the contents of the {@code resources} * attribute) @@ -317,11 +297,6 @@ static boolean isJavaBinaryOrJavaTest(RuleContext ruleContext) { || ruleContext.getRule().getRuleClass().equals("java_test"); } - /** Adds extra runfiles for a {@code java_binary} rule. */ - void addRunfilesForBinary( - RuleContext ruleContext, Artifact launcher, Runfiles.Builder runfilesBuilder) - throws RuleErrorException; - /** Adds extra runfiles for a {@code java_library} rule. */ void addRunfilesForLibrary(RuleContext ruleContext, Runfiles.Builder runfilesBuilder); @@ -349,44 +324,6 @@ void collectTargetsTreatedAsDeps( String addCoverageSupport(JavaCompilationHelper helper, Artifact executable) throws InterruptedException, RuleErrorException; - /** Return the JVM flags to be used in a Java binary. */ - Iterable getJvmFlags( - RuleContext ruleContext, ImmutableList srcsArtifacts, List userJvmFlags); - - /** Translates XMB messages to translations artifact suitable for Java targets. */ - ImmutableList translate(RuleContext ruleContext, List messages); - - /** - * Get the launcher artifact for a java binary, creating the necessary actions for it. - * - * @param ruleContext The rule context - * @param common The common helper class. - * @param deployArchiveBuilder the builder to construct the deploy archive action (mutable). - * @param unstrippedDeployArchiveBuilder the builder to construct the unstripped deploy archive - * action (mutable). - * @param runfilesBuilder the builder to construct the list of runfiles (mutable). - * @param jvmFlags the list of flags to pass to the JVM when running the Java binary (mutable). - * @param attributesBuilder the builder to construct the list of attributes of this target - * (mutable). - * @param ccToolchain to be used to build the launcher - * @param featureConfiguration to be used to configure the cc toolchain - * @return the launcher and unstripped launcher as an artifact pair. If shouldStrip is false, then - * they will be the same. - * @throws InterruptedException - */ - Pair getLauncher( - final RuleContext ruleContext, - final JavaCommon common, - DeployArchiveBuilder deployArchiveBuilder, - DeployArchiveBuilder unstrippedDeployArchiveBuilder, - Builder runfilesBuilder, - List jvmFlags, - JavaTargetAttributes.Builder attributesBuilder, - boolean shouldStrip, - CcToolchainProvider ccToolchain, - FeatureConfiguration featureConfiguration) - throws InterruptedException, RuleErrorException; - /** * Add a source artifact to a {@link JavaTargetAttributes.Builder}. It is called when a source * artifact is processed but is not matched by default patterns in the {@link @@ -407,9 +344,6 @@ Pair getLauncher( */ PathFragment getDefaultJavaResourcePath(PathFragment path); - /** @return a list of extra arguments to appends to the runfiles support. */ - List getExtraArguments(RuleContext ruleContext, ImmutableList sources); - /** * @return An artifact representing the protobuf-format version of the proguard mapping, or null * if the proguard version doesn't support this. diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaTargetAttributes.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaTargetAttributes.java index fedee3356aa4ef..bbf838f698b2a6 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaTargetAttributes.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaTargetAttributes.java @@ -284,15 +284,6 @@ public Builder addNativeLibrary(Artifact nativeLibrary) { return this; } - @CanIgnoreReturnValue - public Builder addNativeLibraries(Iterable nativeLibraries) { - Preconditions.checkArgument(!built); - for (Artifact nativeLibrary : nativeLibraries) { - addNativeLibrary(nativeLibrary); - } - return this; - } - @CanIgnoreReturnValue public Builder addMessages(Collection messages) { Preconditions.checkArgument(!built); From b512ead27d958b3bc6d2a5b918a4161486f59b3f Mon Sep 17 00:00:00 2001 From: Googler Date: Tue, 4 Jul 2023 05:10:00 -0700 Subject: [PATCH 09/51] Expose `_neverlink` and `_constraints` from native `JavaInfo` As `java_common` methods move to Starlark, `JavaInfo` instances will move bi-directionally between Starlark<->Java. We need to synchronize the native API to be able to recreate a native `JavaInfo` instance in Starlark. PiperOrigin-RevId: 545415994 Change-Id: I5c3af1583b2bf149327fb9e758247243177e6652 --- .../build/lib/rules/java/JavaInfo.java | 21 ++++++++++++++++--- .../starlarkbuildapi/java/JavaInfoApi.java | 9 ++++++++ .../builtins_bzl/common/java/java_info.bzl | 2 ++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfo.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfo.java index adf6d06438bbd8..32e23782660e62 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfo.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfo.java @@ -239,7 +239,7 @@ static T nullIfNone(Object object, Class type) { /** Java constraints (e.g. "android") that are present on the target. */ private final ImmutableList javaConstraints; - // Whether or not this library should be used only for compilation and not at runtime. + // Whether this library should be used only for compilation and not at runtime. private final boolean neverlink; @Nullable @@ -404,7 +404,7 @@ private JavaInfo(StructImpl javaInfo) throws EvalException, TypeException, RuleE JavaSourceJarsProvider.fromStarlarkJavaInfo(javaInfo), extractDirectRuntimeJars(javaInfo), extractNeverLink(javaInfo), - ImmutableList.of(), + extractConstraints(javaInfo), javaInfo.getCreationLocation()); } @@ -420,12 +420,22 @@ private static boolean extractNeverLink(StructImpl javaInfo) throws EvalExceptio return neverlink != null && neverlink; } + private static ImmutableList extractConstraints(StructImpl javaInfo) + throws EvalException { + Object constraints = javaInfo.getValue("_constraints"); + if (constraints == null || constraints == Starlark.NONE) { + return ImmutableList.of(); + } + return Sequence.cast(constraints, String.class, "_constraints").getImmutableList(); + } + @Override public JavaInfoProvider getProvider() { return PROVIDER; } - public Boolean isNeverlink() { + @Override + public boolean isNeverlink() { return neverlink; } @@ -581,6 +591,11 @@ public ImmutableList getJavaConstraints() { return javaConstraints; } + @Override + public Sequence getJavaConstraintsStarlark() { + return StarlarkList.immutableCopyOf(javaConstraints); + } + /** * Gets Provider, check it for not null and call function to get NestedSet<S> from it. * diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaInfoApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaInfoApi.java index 19cd50c4bda00f..fa693977e7b46d 100644 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaInfoApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaInfoApi.java @@ -32,6 +32,12 @@ public interface JavaInfoApi< JavaPluginDataT extends JavaPluginDataApi> extends StructApi, JavaPluginInfoApi { + @StarlarkMethod( + name = "_neverlink", + doc = "Whether this library should be used only for compilation and not at runtime.", + structField = true) + boolean isNeverlink(); + @StarlarkMethod( name = "transitive_runtime_jars", doc = "Returns a transitive set of Jars required on the target's runtime classpath.", @@ -171,6 +177,9 @@ public interface JavaInfoApi< @StarlarkMethod(name = "_compile_time_java_dependencies", documented = false, structField = true) Depset getCompileTimeJavaDependencies(); + @StarlarkMethod(name = "_constraints", documented = false, structField = true) + Sequence getJavaConstraintsStarlark(); + /** Provider class for {@link JavaInfoApi} objects. */ interface JavaInfoProviderApi extends ProviderApi { diff --git a/src/main/starlark/builtins_bzl/common/java/java_info.bzl b/src/main/starlark/builtins_bzl/common/java/java_info.bzl index d0f5c0b425fc87..bcfed54c7e764b 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_info.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_info.bzl @@ -240,6 +240,7 @@ def _javainfo_init( transitive = [dep._compile_time_java_dependencies for dep in exports] + ([depset([compile_jdeps])] if compile_jdeps else []), ), + "_constraints": [], } if _java_common_internal._google_legacy_api_enabled(): cc_info = cc_common.merge_cc_infos( @@ -310,6 +311,7 @@ JavaInfo, _new_javainfo = provider( "_compile_time_java_dependencies": "internal API, do not use", "_neverlink": "internal API, do not use", "_plugin_info": "internal API, do not use", + "_constraints": "internal API, do not use", }, init = _javainfo_init, ) From 67b71cda6c5f4a2779b67b532a67f25b2ecd383f Mon Sep 17 00:00:00 2001 From: Googler Date: Tue, 4 Jul 2023 05:14:34 -0700 Subject: [PATCH 10/51] Move `announce` option to graveyard. PiperOrigin-RevId: 545416891 Change-Id: I8f9a7be2d2db418dfdf0a35c1fd2d63cb755539b --- .../devtools/build/lib/bazel/rules/BazelRulesModule.java | 9 +++++++++ .../build/lib/buildtool/BuildRequestOptions.java | 9 --------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java index e494a37d720f23..a0014f9250b547 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java @@ -403,6 +403,15 @@ public static class BuildGraveyardOptions extends OptionsBase { effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, help = "Deprecated no-op.") public boolean showArtifacts; + + @Option( + name = "announce", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, + help = "Deprecated. No-op.", + deprecationWarning = "This option is now deprecated and is a no-op") + public boolean announce; } /** This is where deprecated Bazel-specific options only used by the build command go to die. */ diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java index 6485cb0a2c3615..c8b0813a4cbdb1 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java @@ -203,15 +203,6 @@ public class BuildRequestOptions extends OptionsBase { + " under the threshold.") public int maxResultTargets; - @Option( - name = "announce", - defaultValue = "false", - documentationCategory = OptionDocumentationCategory.LOGGING, - effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, - help = "Deprecated. No-op.", - deprecationWarning = "This option is now deprecated and is a no-op") - public boolean announce; - @Option( name = "symlink_prefix", defaultValue = "null", From 228f1bf8e480c3be6d77f47d387f9396a79086dd Mon Sep 17 00:00:00 2001 From: Googler Date: Tue, 4 Jul 2023 05:54:33 -0700 Subject: [PATCH 11/51] Move `print_workspace_in_output_paths_if_needed` to the graveyard. PiperOrigin-RevId: 545423583 Change-Id: Id98baf7e962c17538a3bfef529f257fdec4bae29 --- .../lib/bazel/rules/BazelRulesModule.java | 8 + .../lib/buildtool/BuildRequestOptions.java | 13 -- .../lib/buildtool/BuildResultPrinter.java | 5 +- .../buildtool/OutputDirectoryLinksUtils.java | 9 +- .../lib/buildtool/PathPrettyPrinter.java | 11 +- .../lib/runtime/commands/RunCommand.java | 5 +- .../lib/runtime/commands/TestCommand.java | 1 - .../lib/buildtool/BuildResultTestCase.java | 150 ------------------ 8 files changed, 14 insertions(+), 188 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java index a0014f9250b547..67916fe9b6533e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java @@ -412,6 +412,14 @@ public static class BuildGraveyardOptions extends OptionsBase { help = "Deprecated. No-op.", deprecationWarning = "This option is now deprecated and is a no-op") public boolean announce; + + @Option( + name = "print_workspace_in_output_paths_if_needed", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = {OptionEffectTag.TERMINAL_OUTPUT}, + help = "Deprecated no-op.") + public boolean printWorkspaceInOutputPathsIfNeeded; } /** This is where deprecated Bazel-specific options only used by the build command go to die. */ diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java index c8b0813a4cbdb1..77cc4a43931076 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java @@ -374,19 +374,6 @@ public boolean useTopLevelTargetsForSymlinks() { + " This flag is not planned to be enabled by default, and should not be relied on.") public boolean experimentalCreatePySymlinks; - @Option( - name = "print_workspace_in_output_paths_if_needed", - defaultValue = "false", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - effectTags = {OptionEffectTag.TERMINAL_OUTPUT}, - help = - "If enabled, when the current working directory is deeper than the workspace (for" - + " example, when running from /foo instead of ), printed" - + " output paths include the absolute path to the workspace (for example," - + " /-bin/foo/binary instead of " - + "-bin/foo/binary).") - public boolean printWorkspaceInOutputPathsIfNeeded; - @Option( name = "use_action_cache", defaultValue = "true", diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildResultPrinter.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildResultPrinter.java index 03f03d4b3757d5..779cdd3f6b7ffc 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildResultPrinter.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildResultPrinter.java @@ -95,10 +95,7 @@ private boolean outputTargets( runtime.getRuleClassProvider().getSymlinkDefinitions(), request.getBuildOptions().getSymlinkPrefix(productName), productName, - env.getWorkspace(), - request.getBuildOptions().printWorkspaceInOutputPathsIfNeeded - ? env.getWorkingDirectory() - : env.getWorkspace()); + env.getWorkspace()); OutErr outErr = request.getOutErr(); // Splits aspects based on whether they are validation aspects. diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/OutputDirectoryLinksUtils.java b/src/main/java/com/google/devtools/build/lib/buildtool/OutputDirectoryLinksUtils.java index ac93bf2059f9a8..47ba5a408ff0f3 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/OutputDirectoryLinksUtils.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/OutputDirectoryLinksUtils.java @@ -179,14 +179,9 @@ public static PathPrettyPrinter getPathPrettyPrinter( Iterable symlinkDefinitions, String symlinkPrefix, String productName, - Path workspaceDirectory, - Path workingDirectory) { + Path workspaceDirectory) { return new PathPrettyPrinter( - getAllLinkDefinitions(symlinkDefinitions), - symlinkPrefix, - productName, - workspaceDirectory, - workingDirectory); + getAllLinkDefinitions(symlinkDefinitions), symlinkPrefix, productName, workspaceDirectory); } private static void removeAllSymlinks( diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/PathPrettyPrinter.java b/src/main/java/com/google/devtools/build/lib/buildtool/PathPrettyPrinter.java index 064df933227741..c1d4e7d71cbadb 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/PathPrettyPrinter.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/PathPrettyPrinter.java @@ -29,7 +29,6 @@ public final class PathPrettyPrinter { private final String symlinkPrefix; private final String productName; private final Path workspaceDirectory; - private final Path workingDirectory; /** * Creates a path pretty printer, immediately resolving the symlink definitions by reading the @@ -39,12 +38,10 @@ public final class PathPrettyPrinter { ImmutableList symlinkDefinitions, String symlinkPrefix, String productName, - Path workspaceDirectory, - Path workingDirectory) { + Path workspaceDirectory) { this.symlinkPrefix = symlinkPrefix; this.productName = productName; this.workspaceDirectory = workspaceDirectory; - this.workingDirectory = workingDirectory; this.resolvedSymlinks = resolve(symlinkDefinitions); } @@ -87,11 +84,7 @@ public PathFragment getPrettyPath(PathFragment file) { PathFragment linkFragment = e.getKey(); PathFragment linkTarget = e.getValue(); if (file.startsWith(linkTarget)) { - PathFragment outputLink = - workingDirectory.equals(workspaceDirectory) - ? linkFragment - : workspaceDirectory.getRelative(linkFragment).asFragment(); - return outputLink.getRelative(file.relativeTo(linkTarget)); + return linkFragment.getRelative(file.relativeTo(linkTarget)); } } diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java index fa0363bf4208e0..6576e2eca47d4a 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java @@ -622,10 +622,7 @@ private static void constructCommandLine( runtime.getRuleClassProvider().getSymlinkDefinitions(), requestOptions.getSymlinkPrefix(productName), productName, - env.getWorkspace(), - requestOptions.printWorkspaceInOutputPathsIfNeeded - ? env.getWorkingDirectory() - : env.getWorkspace()); + env.getWorkspace()); PathFragment prettyExecutablePath = prettyPrinter.getPrettyPath(executable.getPath().asFragment()); diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/TestCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/TestCommand.java index 43f006d9196458..cf772847db75fe 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/commands/TestCommand.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/TestCommand.java @@ -255,7 +255,6 @@ private static TestLogPathFormatter makeTestLogPathFormatter( runtime.getRuleClassProvider().getSymlinkDefinitions(), requestOptions.getSymlinkPrefix(productName), productName, - env.getWorkspace(), env.getWorkspace()); return path -> pathPrettyPrinter.getPrettyPath(path.asFragment()).getPathString(); } diff --git a/src/test/java/com/google/devtools/build/lib/buildtool/BuildResultTestCase.java b/src/test/java/com/google/devtools/build/lib/buildtool/BuildResultTestCase.java index 65359679eeb7a1..b50e80e7538040 100644 --- a/src/test/java/com/google/devtools/build/lib/buildtool/BuildResultTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/buildtool/BuildResultTestCase.java @@ -202,156 +202,6 @@ public void testFailedTargetWithSaveTemps() throws Exception { assertThat(stderr).contains("See temp at blaze-bin/bad_clib/_objs/bad_clib/badlib.pic.ii\n"); } - @Test - public void testSymlinkOutputAtCwdUnderWorkspace_withFlagOn() throws Exception { - write("my_clib/BUILD", "cc_library(name='my_clib', srcs=['myclib.cc'])\n"); - write("my_clib/myclib.cc", "void f() {}"); - - addOptions( - "--print_workspace_in_output_paths_if_needed", - "--client_cwd=" + getWorkspace().getChild("my_clib").getPathString()); - build(false, "no-error", "//my_clib"); - - String stderr = recOutErr.errAsLatin1(); - assertThat(stderr).contains("Target //my_clib:my_clib up-to-date:\n"); - assertThat(stderr) - .contains(getWorkspace().getRelative("blaze-bin/my_clib/libmy_clib.so") + "\n"); - assertThat(stderr) - .contains(getWorkspace().getRelative("blaze-bin/my_clib/libmy_clib.a") + "\n"); - } - - @Test - public void testSymlinkOutputAtCwdUnderWorkspace_withFlagOff() throws Exception { - write("my_clib/BUILD", "cc_library(name='my_clib', srcs=['myclib.cc'])\n"); - write("my_clib/myclib.cc", "void f() {}"); - - addOptions( - "--noprint_workspace_in_output_paths_if_needed", - "--client_cwd=" + getWorkspace().getChild("my_clib").getPathString()); - build(false, "no-error", "//my_clib"); - - String stderr = recOutErr.errAsLatin1(); - assertThat(stderr).contains("Target //my_clib:my_clib up-to-date:\n"); - assertThat(stderr) - .doesNotContain(getWorkspace().getRelative("blaze-bin/my_clib/libmy_clib.so") + "\n"); - assertThat(stderr) - .doesNotContain(getWorkspace().getRelative("blaze-bin/my_clib/libmy_clib.a") + "\n"); - assertThat(stderr).contains("blaze-bin/my_clib/libmy_clib.so\n"); - assertThat(stderr).contains("blaze-bin/my_clib/libmy_clib.a\n"); - } - - @Test - public void testSymlinkOutputAtCwdEqualWorkspace_withFlagOn() throws Exception { - write("my_clib/BUILD", "cc_library(name='my_clib', srcs=['myclib.cc'])\n"); - write("my_clib/myclib.cc", "void f() {}"); - - addOptions("--print_workspace_in_output_paths_if_needed"); - build(false, "no-error", "//my_clib"); - - String stderr = recOutErr.errAsLatin1(); - assertThat(stderr).contains("Target //my_clib:my_clib up-to-date:\n"); - assertThat(stderr) - .doesNotContain(getWorkspace().getRelative("blaze-bin/my_clib/libmy_clib.so") + "\n"); - assertThat(stderr) - .doesNotContain(getWorkspace().getRelative("blaze-bin/my_clib/libmy_clib.a") + "\n"); - assertThat(stderr).contains("blaze-bin/my_clib/libmy_clib.so\n"); - assertThat(stderr).contains("blaze-bin/my_clib/libmy_clib.a\n"); - } - - @Test - public void testSymlinkPrefixAtCwdEqualWorkspace_withFlagOff() throws Exception { - write("my_clib/BUILD", "cc_library(name='my_clib', srcs=['myclib.cc'])\n"); - write("my_clib/myclib.cc", "void f() {}"); - - addOptions("--noprint_workspace_in_output_paths_if_needed"); - build(false, "no-error", "//my_clib"); - - String stderr = recOutErr.errAsLatin1(); - assertThat(stderr).contains("Target //my_clib:my_clib up-to-date:\n"); - assertThat(stderr) - .doesNotContain(getWorkspace().getRelative("blaze-bin/my_clib/libmy_clib.so") + "\n"); - assertThat(stderr) - .doesNotContain(getWorkspace().getRelative("blaze-bin/my_clib/libmy_clib.a") + "\n"); - assertThat(stderr).contains("blaze-bin/my_clib/libmy_clib.so\n"); - assertThat(stderr).contains("blaze-bin/my_clib/libmy_clib.a\n"); - } - - @Test - public void testSeeTempAtCwdUnderWorkspace_withFlagOn() throws Exception { - write("bad_clib/BUILD", "cc_library(name='bad_clib', srcs=['badlib.cc'])\n"); - // trigger a warning to make the build fail: - // "control reaches end of non-void function [-Werror,-Wreturn-type]" - write("bad_clib/badlib.cc", "int f() { }"); - - // We need to set --keep_going so that the temps get built even though the compilation fails. - addOptions( - "--save_temps", - "--keep_going", - "--print_workspace_in_output_paths_if_needed", - "--client_cwd=" + getWorkspace().getChild("bad_clib").getPathString()); - build(true, "compilation of rule '//bad_clib:bad_clib' failed", "//bad_clib"); - - String stderr = recOutErr.errAsLatin1(); - assertThat(stderr).contains("Target //bad_clib:bad_clib failed to build"); - assertThat(stderr) - .contains( - "See temp at " - + getWorkspace().getRelative("blaze-bin/bad_clib/_objs/bad_clib/badlib.pic.ii") - + "\n"); - } - - @Test - public void testSeeTempAtCwdUnderWorkspace_withFlagOff() throws Exception { - write("bad_clib/BUILD", "cc_library(name='bad_clib', srcs=['badlib.cc'])\n"); - // trigger a warning to make the build fail: - // "control reaches end of non-void function [-Werror,-Wreturn-type]" - write("bad_clib/badlib.cc", "int f() { }"); - - // We need to set --keep_going so that the temps get built even though the compilation fails. - addOptions( - "--save_temps", - "--keep_going", - "--noprint_workspace_in_output_paths_if_needed", - "--client_cwd=" + getWorkspace().getChild("bad_clib").getPathString()); - build(true, "compilation of rule '//bad_clib:bad_clib' failed", "//bad_clib"); - - String stderr = recOutErr.errAsLatin1(); - assertThat(stderr).contains("Target //bad_clib:bad_clib failed to build"); - assertThat(stderr).contains("See temp at blaze-bin/bad_clib/_objs/bad_clib/badlib.pic.ii\n"); - } - - @Test - public void testSeeTempAtCwdEqualWorkspace_withFlagOn() throws Exception { - write("bad_clib/BUILD", "cc_library(name='bad_clib', srcs=['badlib.cc'])\n"); - // trigger a warning to make the build fail: - // "control reaches end of non-void function [-Werror,-Wreturn-type]" - write("bad_clib/badlib.cc", "int f() { }"); - - // We need to set --keep_going so that the temps get built even though the compilation fails. - addOptions("--save_temps", "--keep_going", "--print_workspace_in_output_paths_if_needed"); - build(true, "compilation of rule '//bad_clib:bad_clib' failed", "//bad_clib"); - - String stderr = recOutErr.errAsLatin1(); - assertThat(stderr).contains("Target //bad_clib:bad_clib failed to build"); - assertThat(stderr).contains("See temp at blaze-bin/bad_clib/_objs/bad_clib/badlib.pic.ii\n"); - } - - @Test - public void testSeeTempAtCwdEqualWorkspace_withFlagOff() throws Exception { - write("bad_clib/BUILD", "cc_library(name='bad_clib', srcs=['badlib.cc'])\n"); - // trigger a warning to make the build fail: - // "control reaches end of non-void function [-Werror,-Wreturn-type]" - write("bad_clib/badlib.cc", "int f() { }"); - - // We need to set --keep_going so that the temps get built even though the compilation fails. - addOptions("--save_temps", "--keep_going", "--noprint_workspace_in_output_paths_if_needed"); - build(true, "compilation of rule '//bad_clib:bad_clib' failed", "//bad_clib"); - - String stderr = recOutErr.errAsLatin1(); - assertThat(stderr).contains("Target //bad_clib:bad_clib failed to build"); - assertThat(stderr).contains("See temp at blaze-bin/bad_clib/_objs/bad_clib/badlib.pic.ii\n"); - } - // Concrete implementations of this abstract test: /** Tests with 1 job. */ From cbfcab91c90e3751eb9a50ac9b6f94e408d27d28 Mon Sep 17 00:00:00 2001 From: Googler Date: Tue, 4 Jul 2023 06:09:46 -0700 Subject: [PATCH 12/51] Deprecate `--experimental_multi_cpu`. PiperOrigin-RevId: 545426994 Change-Id: I82492988aa3dff03aea190b5f4d2d6928e413a7b --- .../build/lib/bazel/rules/BazelRulesModule.java | 12 ++++++++++++ .../build/lib/buildtool/BuildRequestOptions.java | 12 ------------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java index 67916fe9b6533e..730c55ffb0a9cf 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java @@ -420,6 +420,18 @@ public static class BuildGraveyardOptions extends OptionsBase { effectTags = {OptionEffectTag.TERMINAL_OUTPUT}, help = "Deprecated no-op.") public boolean printWorkspaceInOutputPathsIfNeeded; + + @Option( + name = "experimental_multi_cpu", + deprecationWarning = "This flag is a no-op and will be deleted in a future release.", + converter = Converters.CommaSeparatedOptionListConverter.class, + allowMultiple = true, + defaultValue = "null", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, + metadataTags = {OptionMetadataTag.EXPERIMENTAL}, + help = "Deprecated. No-op.") + public List multiCpus; } /** This is where deprecated Bazel-specific options only used by the build command go to die. */ diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java index 77cc4a43931076..cb3f567935c274 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java @@ -251,18 +251,6 @@ public class BuildRequestOptions extends OptionsBase { + "the convenienceSymlinksIdentified entry in the BuildEventProtocol will be empty.") public boolean experimentalConvenienceSymlinksBepEvent; - @Option( - name = "experimental_multi_cpu", - deprecationWarning = "This flag is a no-op and will be deleted in a future release.", - converter = Converters.CommaSeparatedOptionListConverter.class, - allowMultiple = true, - defaultValue = "null", - documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS, - effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, - metadataTags = {OptionMetadataTag.EXPERIMENTAL}, - help = "Deprecated. No-op.") - public List multiCpus; - @Option( name = "output_tree_tracking", oldName = "experimental_output_tree_tracking", From a07e1869b5bd0aef2b33791898bc3730996a49b7 Mon Sep 17 00:00:00 2001 From: Googler Date: Tue, 4 Jul 2023 06:12:18 -0700 Subject: [PATCH 13/51] Move `action_cache_store_output_metadata` to graveyard. It has been made a no-op in https://github.com/bazelbuild/bazel/commit/ec6ce5bdeb6c0977eab3f4e3925fb3ccd5a06460 PiperOrigin-RevId: 545427428 Change-Id: Ife1bed961595e468673adf0a9c1d5b6906a22dc1 --- .../build/lib/bazel/rules/BazelRulesModule.java | 12 ++++++++++++ .../build/lib/buildtool/BuildRequestOptions.java | 12 ------------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java index 730c55ffb0a9cf..88f1333b604365 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java @@ -432,6 +432,18 @@ public static class BuildGraveyardOptions extends OptionsBase { metadataTags = {OptionMetadataTag.EXPERIMENTAL}, help = "Deprecated. No-op.") public List multiCpus; + + @Option( + name = "action_cache_store_output_metadata", + oldName = "experimental_action_cache_store_output_metadata", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = { + OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION, + OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS + }, + help = "no-op") + public boolean actionCacheStoreOutputMetadata; } /** This is where deprecated Bazel-specific options only used by the build command go to die. */ diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java index cb3f567935c274..423260ab72d32c 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java @@ -373,18 +373,6 @@ public boolean useTopLevelTargetsForSymlinks() { help = "Whether to use the action cache") public boolean useActionCache; - @Option( - name = "action_cache_store_output_metadata", - oldName = "experimental_action_cache_store_output_metadata", - defaultValue = "false", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - effectTags = { - OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION, - OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS - }, - help = "no-op") - public boolean actionCacheStoreOutputMetadata; - @Option( name = "rewind_lost_inputs", defaultValue = "false", From bc99d4162f5707899603a51e566c827b7da79144 Mon Sep 17 00:00:00 2001 From: Googler Date: Tue, 4 Jul 2023 06:14:44 -0700 Subject: [PATCH 14/51] Move `discard_actions_after_execution` to graveyard. PiperOrigin-RevId: 545428159 Change-Id: Iae2327e89ac99a7e7b0c83d0243955d205306cac --- .../devtools/build/lib/bazel/rules/BazelRulesModule.java | 9 +++++++++ .../build/lib/buildtool/BuildRequestOptions.java | 9 --------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java index 88f1333b604365..12abe69f89cd0b 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java @@ -444,6 +444,15 @@ public static class BuildGraveyardOptions extends OptionsBase { }, help = "no-op") public boolean actionCacheStoreOutputMetadata; + + @Option( + name = "discard_actions_after_execution", + defaultValue = "true", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + metadataTags = OptionMetadataTag.INCOMPATIBLE_CHANGE, + effectTags = {OptionEffectTag.LOSES_INCREMENTAL_STATE}, + help = "This option is deprecated and has no effect.") + public boolean discardActionsAfterExecution; } /** This is where deprecated Bazel-specific options only used by the build command go to die. */ diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java index 423260ab72d32c..5bf358c4382192 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java @@ -383,15 +383,6 @@ public boolean useTopLevelTargetsForSymlinks() { + " prerequisites for rewinding are met (no incrementality, no action cache).") public boolean rewindLostInputs; - @Option( - name = "discard_actions_after_execution", - defaultValue = "true", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - metadataTags = OptionMetadataTag.INCOMPATIBLE_CHANGE, - effectTags = {OptionEffectTag.LOSES_INCREMENTAL_STATE}, - help = "This option is deprecated and has no effect.") - public boolean discardActionsAfterExecution; - @Option( name = "incompatible_skip_genfiles_symlink", defaultValue = "true", From 4034ae021b28442137cb36c87d83b149c067e930 Mon Sep 17 00:00:00 2001 From: Googler Date: Tue, 4 Jul 2023 06:50:29 -0700 Subject: [PATCH 15/51] Delete unused native API `java_common.provider` PiperOrigin-RevId: 545434528 Change-Id: I16e3570389acf6d84a975c84034958c2b946fd22 --- .../build/lib/rules/java/JavaStarlarkCommon.java | 5 ----- .../build/lib/starlarkbuildapi/java/JavaCommonApi.java | 9 --------- 2 files changed, 14 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java index 720d9e9eab7808..1faa48cb9cd9f6 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java @@ -80,11 +80,6 @@ public JavaStarlarkCommon(JavaSemantics javaSemantics) { this.javaSemantics = javaSemantics; } - @Override - public Provider getJavaProvider() { - return JavaInfo.PROVIDER; - } - @Override public JavaInfo createJavaCompileAction( StarlarkRuleContext starlarkRuleContext, diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java index 2461305e43bf3a..0a7a420a2fdff9 100644 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java @@ -56,15 +56,6 @@ public interface JavaCommonApi< StarlarkActionFactoryT extends StarlarkActionFactoryApi> extends StarlarkValue { - @StarlarkMethod( - name = "provider", - structField = true, - doc = - "Returns the Java declared provider.
" - + "The same value is accessible as JavaInfo.
" - + "Prefer using JavaInfo in new code.") - ProviderApi getJavaProvider(); - @StarlarkMethod( name = "compile", doc = From 9bebac21363c75942cab5e291ee9868ba4cce378 Mon Sep 17 00:00:00 2001 From: Googler Date: Tue, 4 Jul 2023 07:30:36 -0700 Subject: [PATCH 16/51] Move `defer_param_files` to the graveyard. PiperOrigin-RevId: 545442679 Change-Id: I98e4971668f7fc76761cf6c8cd7bac56e12b190f --- .../lib/analysis/config/CoreOptions.java | 12 --------- .../lib/bazel/rules/BazelRulesModule.java | 12 +++++++++ .../lib/analysis/actions/SpawnActionTest.java | 25 ++----------------- 3 files changed, 14 insertions(+), 35 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java b/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java index 0c996c3ee2554f..20b327f07051b4 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java @@ -155,18 +155,6 @@ public class CoreOptions extends FragmentOptions implements Cloneable { help = "Minimum command line length before creating a parameter file.") public int minParamFileSize; - @Option( - name = "defer_param_files", - defaultValue = "true", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - effectTags = { - OptionEffectTag.LOADING_AND_ANALYSIS, - OptionEffectTag.EXECUTION, - OptionEffectTag.ACTION_COMMAND_LINES - }, - help = "This option is deprecated and has no effect and will be removed in the future.") - public boolean deferParamFiles; - @Option( name = "experimental_extended_sanity_checks", defaultValue = "false", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java index 12abe69f89cd0b..0ce586fa345f56 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java @@ -453,6 +453,18 @@ public static class BuildGraveyardOptions extends OptionsBase { effectTags = {OptionEffectTag.LOSES_INCREMENTAL_STATE}, help = "This option is deprecated and has no effect.") public boolean discardActionsAfterExecution; + + @Option( + name = "defer_param_files", + defaultValue = "true", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = { + OptionEffectTag.LOADING_AND_ANALYSIS, + OptionEffectTag.EXECUTION, + OptionEffectTag.ACTION_COMMAND_LINES + }, + help = "This option is deprecated and has no effect and will be removed in the future.") + public boolean deferParamFiles; } /** This is where deprecated Bazel-specific options only used by the build command go to die. */ diff --git a/src/test/java/com/google/devtools/build/lib/analysis/actions/SpawnActionTest.java b/src/test/java/com/google/devtools/build/lib/analysis/actions/SpawnActionTest.java index 24da5e03ceedf9..b6cd9a53dbe1f2 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/actions/SpawnActionTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/actions/SpawnActionTest.java @@ -135,7 +135,7 @@ public void testExecutionInfo_fromExecutionPlatform() throws Exception { Label.parseCanonicalUnchecked("//target"), new Location("dummy-file", 0, 0), /* targetKind= */ "dummy-kind", - /* mnemonic= */ "dummy-configuration-mnemonic", + /* buildConfigurationMnemonic= */ "dummy-configuration-mnemonic", /* configurationChecksum= */ "dummy-configuration", new BuildConfigurationEvent( BuildEventStreamProtos.BuildEventId.getDefaultInstance(), @@ -231,7 +231,7 @@ public void testBuilderWithJarExecutable() throws Exception { @Test public void testBuilderWithJarExecutableAndParameterFile2() throws Exception { - useConfiguration("--min_param_file_size=0", "--defer_param_files"); + useConfiguration("--min_param_file_size=0"); collectingAnalysisEnvironment = new AnalysisTestUtil.CollectingAnalysisEnvironment(getTestAnalysisEnvironment()); Artifact output = getBinArtifactWithNoOwner("output"); @@ -320,27 +320,6 @@ public void testMultipleCommandLines() throws Exception { assertThat(action.getArguments()).containsExactly("/bin/xxx", "arg1", "arg2").inOrder(); } - @Test - public void testGetArgumentsWithParameterFiles() throws Exception { - useConfiguration("--min_param_file_size=0", "--nodefer_param_files"); - Artifact input = getSourceArtifact("input"); - Artifact output = getBinArtifactWithNoOwner("output"); - SpawnAction action = - builder() - .addInput(input) - .addOutput(output) - .setExecutable(scratch.file("/bin/xxx").asFragment()) - .addCommandLine( - CommandLine.of(ImmutableList.of("arg1")), - ParamFileInfo.builder(ParameterFileType.UNQUOTED).build()) - .addCommandLine( - CommandLine.of(ImmutableList.of("arg2")), - ParamFileInfo.builder(ParameterFileType.UNQUOTED).build()) - .build(nullOwnerWithTargetConfig(), targetConfig); - // getArguments returns all arguments, regardless whether some go in parameter files or not - assertThat(action.getArguments()).containsExactly("/bin/xxx", "arg1", "arg2").inOrder(); - } - @Test public void testExtraActionInfo() throws Exception { SpawnAction action = createCopyFromWelcomeToDestination(ImmutableMap.of()); From 79a77ee100ae9ee005bb86dd2527cfc75096069e Mon Sep 17 00:00:00 2001 From: Googler Date: Tue, 4 Jul 2023 07:50:21 -0700 Subject: [PATCH 17/51] Delete unused native API `java_common.JavaPluginInfo` PiperOrigin-RevId: 545446171 Change-Id: I1b754fac49d776431ff95c1e59f614a974b5ece8 --- .../build/lib/rules/java/JavaStarlarkCommon.java | 5 ----- .../build/lib/starlarkbuildapi/java/JavaCommonApi.java | 9 --------- 2 files changed, 14 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java index 1faa48cb9cd9f6..2154cd970c414d 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java @@ -306,11 +306,6 @@ public JavaInfo makeNonStrict(JavaInfo javaInfo) { .build(); } - @Override - public ProviderApi getJavaPluginProvider() { - return JavaPluginInfo.PROVIDER; - } - @Override public Provider getJavaToolchainProvider() { return JavaToolchainProvider.PROVIDER; diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java index 0a7a420a2fdff9..db3b3f4c375d3a 100644 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java @@ -529,15 +529,6 @@ JavaInfoT mergeJavaProviders( }) JavaInfoT makeNonStrict(JavaInfoT javaInfo); - @StarlarkMethod( - name = "JavaPluginInfo", - doc = - "The key used to retrieve the provider that contains information about the Java " - + "plugins. The same value is accessible as JavaPluginInfo.
" - + "Prefer using JavaPluginInfo in new code.", - structField = true) - ProviderApi getJavaPluginProvider(); - @StarlarkMethod( name = "JavaToolchainInfo", doc = From 4627d6079f17d01a73830cda799a0bfe308a7e4e Mon Sep 17 00:00:00 2001 From: Googler Date: Tue, 4 Jul 2023 08:35:27 -0700 Subject: [PATCH 18/51] Fix a couple of cquery/aquery latent bugs caused by delegation. * cquery / aquery stops rdep traversals by effectively comparing ConfiguredTargetValues. When there are delegating nodes, they look like duplicates and cause traversals to stop prematurely. Now skips such nodes during rdeps traversal. * Implicit deps are stored in ConfiguredTargets using their actual configurations and not their "graph" configurations. Uses the ConfiguredTargetKey of the actual ConfiguredTargetValue when checking these. PiperOrigin-RevId: 545455484 Change-Id: I64cab9dd364d03f751634125751cbc3f2819004e --- .../query2/PostAnalysisQueryEnvironment.java | 87 ++++++++++++++++++- 1 file changed, 84 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/query2/PostAnalysisQueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/PostAnalysisQueryEnvironment.java index 0b78322dfde195..df15e03e71aa55 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/PostAnalysisQueryEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/query2/PostAnalysisQueryEnvironment.java @@ -13,8 +13,11 @@ // limitations under the License. package com.google.devtools.build.lib.query2; +import static com.google.common.base.MoreObjects.toStringHelper; + import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedSet; @@ -286,7 +289,9 @@ public Collection getReverseDeps(Iterable targets, QueryExpressionContext< targetsByKey.put(getConfiguredTargetKey(target).toKey(), target); } Map>> reverseDepsByKey = - targetifyValues(targetsByKey, graph.getReverseDeps(targetsByKey.keySet())); + targetifyValues( + targetsByKey, + skipDelegatingAncestors(graph.getReverseDeps(targetsByKey.keySet())).asMap()); if (targetsByKey.size() != reverseDepsByKey.size()) { Iterable missingTargets = Sets.difference(targetsByKey.keySet(), reverseDepsByKey.keySet()).stream() @@ -322,6 +327,74 @@ private Collection filterReverseDeps( return result; } + /** + * Expands any delegating ancestors when computing reverse dependencies. + * + *

The {@link ConfiguredTargetKey} graph contains delegation entries where instead of + * computing its own value, it delegates to a child with the same labels but a different + * configuration. This causes problems in reverse dependency traversal because traversal stops at + * duplicate values. The delegating parent has the same value as the delegate child. + * + *

This method replaces any delegating ancestor in the set of reverse dependencies with the + * reverse dependencies of the ancestor. + */ + private ImmutableListMultimap skipDelegatingAncestors( + Map> reverseDeps) throws InterruptedException { + var result = ImmutableListMultimap.builder(); + for (Map.Entry> entry : reverseDeps.entrySet()) { + SkyKey child = entry.getKey(); + Iterable rdeps = entry.getValue(); + Set unwoundRdeps = unwindReverseDependencyDelegationLayersIfFound(child, rdeps); + result.putAll(child, unwoundRdeps == null ? rdeps : unwoundRdeps); + } + return result.build(); + } + + @Nullable + private Set unwindReverseDependencyDelegationLayersIfFound( + SkyKey child, Iterable rdeps) throws InterruptedException { + // Most rdeps will not be delegating. Performs an optimistic pass that avoids copying. + boolean foundDelegatingRdep = false; + for (SkyKey rdep : rdeps) { + if (!rdep.functionName().equals(SkyFunctions.CONFIGURED_TARGET)) { + continue; + } + ConfiguredTargetKey actualParentKey = getConfiguredTargetKey(getValueFromKey(rdep)); + if (actualParentKey.equals(child)) { + // The parent has the same value as the child because it is delegating. + foundDelegatingRdep = true; + break; + } + } + if (!foundDelegatingRdep) { + return null; + } + var logicalParents = new HashSet(); + unwindReverseDependencyDelegationLayers(child, rdeps, logicalParents); + return logicalParents; + } + + private void unwindReverseDependencyDelegationLayers( + SkyKey child, Iterable rdeps, Set output) throws InterruptedException { + // Checks the value of each rdep to see if it is delegating to `child`. If so, fetches its rdeps + // and processes those, applying the same expansion as needed. + for (SkyKey rdep : rdeps) { + if (!rdep.functionName().equals(SkyFunctions.CONFIGURED_TARGET)) { + output.add(rdep); + continue; + } + ConfiguredTargetKey actualParentKey = getConfiguredTargetKey(getValueFromKey(rdep)); + if (!actualParentKey.equals(child)) { + output.add(rdep); + continue; + } + // Otherwise `rdep` is delegating to child and needs to be unwound. + Iterable rdepParents = graph.getReverseDeps(ImmutableList.of(rdep)).get(rdep); + // Applies this recursively in case there are multiple layers of delegation. + unwindReverseDependencyDelegationLayers(child, rdepParents, output); + } + } + /** * @param target source target * @param deps next level of deps to filter @@ -416,7 +489,6 @@ private ImmutableList> targetifyValues( continue; } if (key.functionName().equals(SkyFunctions.CONFIGURED_TARGET)) { - ConfiguredTargetKey ctkey = (ConfiguredTargetKey) key.argument(); T dependency = getValueFromKey(key); Preconditions.checkState( dependency != null, @@ -426,7 +498,8 @@ private ImmutableList> targetifyValues( + " configurability team.", key); - boolean implicit = implicitDeps == null || implicitDeps.contains(ctkey); + boolean implicit = + implicitDeps == null || implicitDeps.contains(getConfiguredTargetKey(dependency)); values.add(new ClassifiedDependency<>(dependency, implicit)); knownCtDeps.add(key); } else if (settings.contains(Setting.INCLUDE_ASPECTS) @@ -486,6 +559,14 @@ private ClassifiedDependency(T dependency, boolean implicit) { this.implicit = implicit; this.dependency = dependency; } + + @Override + public String toString() { + return toStringHelper(this) + .add("implicit", implicit) + .add("dependency", dependency) + .toString(); + } } private static ImmutableList getDependencies( From 6b90bf4462bf1cd3dfbc23d51a1c90f6d6841d6d Mon Sep 17 00:00:00 2001 From: Googler Date: Tue, 4 Jul 2023 13:42:29 -0700 Subject: [PATCH 19/51] Allow Error Prone plugins to use `common.regex` PiperOrigin-RevId: 545499378 Change-Id: I4ffbe627fa0ee69d95bbe7bd959bf855e611439b --- .../com/google/devtools/build/buildjar/javac/BlazeJavacMain.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavacMain.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavacMain.java index b5e08f3f45b779..062c22dc443361 100644 --- a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavacMain.java +++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/BlazeJavacMain.java @@ -401,6 +401,7 @@ protected Class findClass(String name) throws ClassNotFoundException { || name.startsWith("com.google.common.collect.") || name.startsWith("com.google.common.base.") || name.startsWith("com.google.common.graph.") + || name.startsWith("com.google.common.regex.") || name.startsWith("org.checkerframework.shaded.dataflow.") || name.startsWith("org.checkerframework.errorprone.dataflow.") || name.startsWith("com.sun.source.") From 0dd6a4994d45fe9b5770eef6b4ac4f49ead26140 Mon Sep 17 00:00:00 2001 From: Googler Date: Tue, 4 Jul 2023 23:07:53 -0700 Subject: [PATCH 20/51] Implement `run_ijar` in Starlark and delete the corresponding native API PiperOrigin-RevId: 545571444 Change-Id: Ibbd07cfae40ed0c1f3c2f67f9bfcd10b8b205765 --- .../lib/rules/java/JavaInfoBuildHelper.java | 36 --------------- .../lib/rules/java/JavaStarlarkCommon.java | 22 --------- .../lib/rules/java/JavaToolchainProvider.java | 1 + .../starlarkbuildapi/java/JavaCommonApi.java | 46 ------------------- .../JavaToolchainStarlarkApiProviderApi.java | 6 +++ .../java_common_internal_for_builtins.bzl | 26 ++++++++--- .../builtins_bzl/common/java/java_helper.bzl | 8 ++++ .../lib/analysis/AutoExecGroupsTest.java | 6 ++- 8 files changed, 39 insertions(+), 112 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfoBuildHelper.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfoBuildHelper.java index 634a5840e59f15..754a838458fb06 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfoBuildHelper.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfoBuildHelper.java @@ -43,7 +43,6 @@ import java.util.List; import java.util.Objects; import java.util.stream.Stream; -import javax.annotation.Nullable; import net.starlark.java.eval.EvalException; import net.starlark.java.eval.Starlark; import net.starlark.java.eval.StarlarkThread; @@ -316,41 +315,6 @@ private static List tokenize(List input) throws EvalException { return output; } - public Artifact buildIjar( - StarlarkActionFactory actions, - Artifact inputJar, - @Nullable Artifact outputJar, - @Nullable Label targetLabel, - JavaToolchainProvider javaToolchain, - String execGroup) - throws EvalException { - Artifact interfaceJar; - if (outputJar != null) { - interfaceJar = outputJar; - } else { - String ijarBasename = FileSystemUtils.removeExtension(inputJar.getFilename()) + "-ijar.jar"; - interfaceJar = actions.declareFile(ijarBasename, inputJar); - } - FilesToRunProvider ijarTarget = javaToolchain.getIjar(); - CustomCommandLine.Builder commandLine = - CustomCommandLine.builder().addExecPath(inputJar).addExecPath(interfaceJar); - if (targetLabel != null) { - commandLine.addLabel("--target_label", targetLabel); - } - SpawnAction.Builder actionBuilder = - new SpawnAction.Builder() - .addInput(inputJar) - .addOutput(interfaceJar) - .setExecutable(ijarTarget) - .setProgressMessage("Extracting interface for jar %s", inputJar.getFilename()) - .addCommandLine(commandLine.build()) - .useDefaultShellEnvironment() - .setMnemonic("JavaIjar") - .setExecGroup(execGroup); - actions.registerAction(actionBuilder.build(actions.getActionConstructionContext())); - return interfaceJar; - } - public Artifact stampJar( StarlarkActionFactory actions, Artifact inputJar, diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java index 2154cd970c414d..e011eb3df3cf71 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java @@ -213,28 +213,6 @@ private String getExecGroup(boolean useAutoExecGroups) { } } - @Override - public Artifact runIjar( - StarlarkActionFactory actions, - Artifact jar, - Object output, - Object targetLabel, - JavaToolchainProvider javaToolchain, - StarlarkThread thread) - throws EvalException { - if (output != Starlark.NONE) { - checkPrivateAccess(thread); - } - return JavaInfoBuildHelper.getInstance() - .buildIjar( - actions, - jar, - output != Starlark.NONE ? (Artifact) output : null, - targetLabel != Starlark.NONE ? (Label) targetLabel : null, - javaToolchain, - getExecGroup(actions.getRuleContext().useAutoExecGroups())); - } - @Override public Artifact stampJar( StarlarkActionFactory actions, diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainProvider.java index c154139546b80f..870fc1c2340546 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainProvider.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainProvider.java @@ -376,6 +376,7 @@ public Artifact getTimezoneData() { } /** Returns the ijar executable */ + @Override public FilesToRunProvider getIjar() { return ijar; } diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java index db3b3f4c375d3a..7e0df85cfb5560 100644 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java @@ -318,52 +318,6 @@ JavaInfoT createJavaCompileAction( StarlarkThread thread) throws EvalException, InterruptedException, RuleErrorException; - @StarlarkMethod( - name = "run_ijar", - doc = - "Runs ijar on a jar, stripping it of its method bodies. This helps reduce rebuilding " - + "of dependent jars during any recompiles consisting only of simple changes to " - + "method implementations. The return value is typically passed to " - + "" - + "JavaInfo#compile_jar.", - parameters = { - @Param(name = "actions", named = true, doc = "ctx.actions"), - @Param(name = "jar", positional = false, named = true, doc = "The jar to run ijar on."), - @Param( - name = "output", - positional = false, - named = true, - documented = false, - defaultValue = "None"), - @Param( - name = "target_label", - positional = false, - named = true, - allowedTypes = { - @ParamType(type = Label.class), - @ParamType(type = NoneType.class), - }, - defaultValue = "None", - doc = - "A target label to stamp the jar with. Used for add_dep support. " - + "Typically, you would pass ctx.label to stamp the jar " - + "with the current rule's label."), - @Param( - name = "java_toolchain", - positional = false, - named = true, - doc = "A JavaToolchainInfo to used to find the ijar tool."), - }, - useStarlarkThread = true) - FileApi runIjar( - StarlarkActionFactoryT actions, - FileT jar, - Object output, - Object targetLabel, - JavaToolchainT javaToolchain, - StarlarkThread thread) - throws EvalException; - @StarlarkMethod( name = "stamp_jar", doc = diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaToolchainStarlarkApiProviderApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaToolchainStarlarkApiProviderApi.java index 11cc3ec2e36766..8c11040504f53c 100644 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaToolchainStarlarkApiProviderApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaToolchainStarlarkApiProviderApi.java @@ -84,6 +84,12 @@ public interface JavaToolchainStarlarkApiProviderApi extends StructApi { structField = true) Depset getStarlarkJvmOptions(); + @StarlarkMethod( + name = "ijar", + doc = "A FilesToRunProvider representing the ijar executable.", + structField = true) + FilesToRunProviderApi getIjar(); + @StarlarkMethod( name = "jacocorunner", doc = "The jacocorunner used by the toolchain.", diff --git a/src/main/starlark/builtins_bzl/common/java/java_common_internal_for_builtins.bzl b/src/main/starlark/builtins_bzl/common/java/java_common_internal_for_builtins.bzl index 63484016de6307..3bca5c95308ecd 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_common_internal_for_builtins.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_common_internal_for_builtins.bzl @@ -14,6 +14,9 @@ """ Private utilities for Java compilation support in Starlark. """ +load(":common/java/java_semantics.bzl", "semantics") +load(":common/java/java_helper.bzl", "helper") + _java_common_internal = _builtins.internal.java_common_internal_do_not_use def compile( @@ -154,13 +157,24 @@ def run_ijar( Returns: (File) The output artifact """ - return _java_common_internal.run_ijar( - actions = actions, - jar = jar, - java_toolchain = java_toolchain, - target_label = target_label, - output = output, + if not output: + output = actions.declare_file(helper.basename_without_extension(jar) + "-ijar.jar", sibling = jar) + args = actions.args() + args.add(jar) + args.add(output) + if target_label != None: + args.add("--target_label", target_label) + actions.run( + mnemonic = "JavaIjar", + inputs = [jar], + outputs = [output], + executable = java_toolchain.ijar, + arguments = [args], + progress_message = "Extracting interface for jar %{input}", + toolchain = semantics.JAVA_TOOLCHAIN_TYPE, + use_default_shell_env = True, ) + return output def merge( providers, diff --git a/src/main/starlark/builtins_bzl/common/java/java_helper.bzl b/src/main/starlark/builtins_bzl/common/java/java_helper.bzl index eebc50d57dec67..cee33cd68f02bc 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_helper.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_helper.bzl @@ -330,6 +330,13 @@ def _create_single_jar(ctx, output, *input_depsets): def _shell_quote(s): return "'" + s.replace("'", "'\\''") + "'" +def _basename_without_extension(file): + before, sep, after = file.basename.rpartition(".") + if sep: + return before + else: + return after + helper = struct( collect_all_targets_as_deps = _collect_all_targets_as_deps, filter_launcher_for_target = _filter_launcher_for_target, @@ -352,4 +359,5 @@ helper = struct( test_providers = _test_providers, create_single_jar = _create_single_jar, shell_quote = _shell_quote, + basename_without_extension = _basename_without_extension, ) diff --git a/src/test/java/com/google/devtools/build/lib/analysis/AutoExecGroupsTest.java b/src/test/java/com/google/devtools/build/lib/analysis/AutoExecGroupsTest.java index 2344f17117ff5d..87b41522cc7c4b 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/AutoExecGroupsTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/AutoExecGroupsTest.java @@ -1498,10 +1498,12 @@ public void javaCommonBuildIjar_automaticExecGroupsEnabled_ijarActionsExecuteOnF "custom_rule(name = 'custom_rule_name')"); useConfiguration("--incompatible_auto_exec_groups"); - ImmutableList actions = getActions("//test:custom_rule_name", SpawnAction.class); + ImmutableList actions = + getActions("//test:custom_rule_name").stream() + .filter(action -> action.getMnemonic().equals("JavaIjar")) + .collect(toImmutableList()); assertThat(actions).hasSize(1); - assertThat(actions.get(0).getMnemonic()).isEqualTo("JavaIjar"); assertThat(actions.get(0).getOwner().getExecutionPlatform().label()) .isEqualTo(Label.parseCanonical("//platforms:platform_1")); } From 9b7730cd280db0cc8ad618a5573c1d99dab5a0bf Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 00:35:00 -0700 Subject: [PATCH 21/51] Move `check_fileset_dependencies_recursively` to the graveyard. PiperOrigin-RevId: 545585833 Change-Id: I6c736b55a6f03458e5b3dbf69d28b283a13f3d62 --- .../build/lib/analysis/config/CoreOptions.java | 10 ---------- .../build/lib/bazel/rules/BazelRulesModule.java | 10 ++++++++++ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java b/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java index 20b327f07051b4..954287d66e7f77 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java @@ -499,16 +499,6 @@ public ExecConfigurationDistinguisherSchemeConverter() { + "https://bazel.build/extending/rules#runfiles_features_to_avoid).") public boolean alwaysIncludeFilesToBuildInData; - @Option( - name = "check_fileset_dependencies_recursively", - defaultValue = "true", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - deprecationWarning = - "This flag is a no-op and fileset dependencies are always checked " - + "to ensure correctness of builds.", - effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}) - public boolean checkFilesetDependenciesRecursively; - @Option( name = "experimental_skyframe_native_filesets", defaultValue = "true", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java index 0ce586fa345f56..40351f498680da 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java @@ -465,6 +465,16 @@ public static class BuildGraveyardOptions extends OptionsBase { }, help = "This option is deprecated and has no effect and will be removed in the future.") public boolean deferParamFiles; + + @Option( + name = "check_fileset_dependencies_recursively", + defaultValue = "true", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + deprecationWarning = + "This flag is a no-op and fileset dependencies are always checked " + + "to ensure correctness of builds.", + effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}) + public boolean checkFilesetDependenciesRecursively; } /** This is where deprecated Bazel-specific options only used by the build command go to die. */ From 232d848102a25caef268ca9859a3c5a7787d3ea6 Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 00:51:16 -0700 Subject: [PATCH 22/51] `--build_event_json_file` should imply `--bes_upload_mode=wait_for_upload_complete`. In unknown commit we did the same for `--build_event_binary_file` - similar to the binary file, the JSON file is often used by tools who will then see an incomplete JSON file and fail parsing it. PiperOrigin-RevId: 545589136 Change-Id: I03c0ed867c6b652098a1d5a7e546ce06bc9a4dca --- .../buildeventstream/transports/BuildEventStreamOptions.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/BuildEventStreamOptions.java b/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/BuildEventStreamOptions.java index 6d7e8b6150aeaf..94f5283331197b 100644 --- a/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/BuildEventStreamOptions.java +++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/BuildEventStreamOptions.java @@ -59,9 +59,12 @@ public class BuildEventStreamOptions extends OptionsBase { name = "build_event_json_file", oldName = "experimental_build_event_json_file", defaultValue = "", + implicitRequirements = {"--bes_upload_mode=wait_for_upload_complete"}, documentationCategory = OptionDocumentationCategory.LOGGING, effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, - help = "If non-empty, write a JSON serialisation of the build event protocol to that file.") + help = + "If non-empty, write a JSON serialisation of the build event protocol to that file." + + " This option implies --bes_upload_mode=wait_for_upload_complete.") public String buildEventJsonFile; @Option( From 1e17348da7e45c00cb474390a3b8ed3103b6b5cf Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 01:20:52 -0700 Subject: [PATCH 23/51] Remove `--experimental_throttle_action_cache_check` flag. There is no reason to disable it. PiperOrigin-RevId: 545594575 Change-Id: Ic61f96a9d1aae49b653a1c1206f9f8fd06eca691 --- .../build/lib/analysis/config/CoreOptions.java | 10 ---------- .../build/lib/bazel/rules/BazelRulesModule.java | 11 +++++++++++ .../build/lib/skyframe/SkyframeActionExecutor.java | 5 +---- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java b/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java index 954287d66e7f77..091aa46ac9f040 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java @@ -967,16 +967,6 @@ public OutputPathsConverter() { + " of failing. This is to help use cquery diagnose failures in select.") public boolean debugSelectsAlwaysSucceed; - @Option( - name = "experimental_throttle_action_cache_check", - defaultValue = "true", - converter = BooleanConverter.class, - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - metadataTags = OptionMetadataTag.EXPERIMENTAL, - effectTags = {OptionEffectTag.EXECUTION}, - help = "Whether to throttle the check whether an action is cached.") - public boolean throttleActionCacheCheck; - /** Ways configured targets may provide the {@link Fragment}s they require. */ public enum IncludeConfigFragmentsEnum { /** diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java index 40351f498680da..04d3c0b3d2b057 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java @@ -26,6 +26,7 @@ import com.google.devtools.build.lib.runtime.CommandEnvironment; import com.google.devtools.build.lib.util.ResourceFileLoader; import com.google.devtools.common.options.Converters; +import com.google.devtools.common.options.Converters.BooleanConverter; import com.google.devtools.common.options.Option; import com.google.devtools.common.options.OptionDocumentationCategory; import com.google.devtools.common.options.OptionEffectTag; @@ -475,6 +476,16 @@ public static class BuildGraveyardOptions extends OptionsBase { + "to ensure correctness of builds.", effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}) public boolean checkFilesetDependenciesRecursively; + + @Option( + name = "experimental_throttle_action_cache_check", + defaultValue = "true", + converter = BooleanConverter.class, + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + metadataTags = OptionMetadataTag.EXPERIMENTAL, + effectTags = {OptionEffectTag.EXECUTION}, + help = "no-op") + public boolean throttleActionCacheCheck; } /** This is where deprecated Bazel-specific options only used by the build command go to die. */ diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java index f15cd79c454fa5..9920a282261a5f 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java @@ -308,10 +308,7 @@ void prepareForExecution( freeDiscoveredInputsAfterExecution = !trackIncrementalState && options.getOptions(CoreOptions.class).actionListeners.isEmpty(); - this.cacheHitSemaphore = - options.getOptions(CoreOptions.class).throttleActionCacheCheck - ? new Semaphore(ResourceUsage.getAvailableProcessors()) - : null; + this.cacheHitSemaphore = new Semaphore(ResourceUsage.getAvailableProcessors()); this.actionExecutionSemaphore = buildRequestOptions.useSemaphoreForJobs ? new Semaphore(buildRequestOptions.jobs) : null; From 167e133457fd38a183651753b27f89d07ee30fb1 Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 02:34:36 -0700 Subject: [PATCH 24/51] Implement `stamp_jar` in Starlark and delete the corresponding native API PiperOrigin-RevId: 545609209 Change-Id: I17bdf8e64f0411b31a263a6d5d6a8b60284ab4a0 --- .../lib/rules/java/JavaInfoBuildHelper.java | 36 ------------------- .../lib/rules/java/JavaStarlarkCommon.java | 16 --------- .../starlarkbuildapi/java/JavaCommonApi.java | 35 ------------------ .../builtins_bzl/common/java/java_common.bzl | 22 +++++++++--- .../lib/analysis/AutoExecGroupsTest.java | 8 +++-- 5 files changed, 23 insertions(+), 94 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfoBuildHelper.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfoBuildHelper.java index 754a838458fb06..133a0d4c135c45 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfoBuildHelper.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfoBuildHelper.java @@ -24,21 +24,16 @@ import com.google.common.collect.Streams; import com.google.devtools.build.lib.actions.ActionRegistry; import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.analysis.FilesToRunProvider; import com.google.devtools.build.lib.analysis.actions.ActionConstructionContext; -import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; -import com.google.devtools.build.lib.analysis.actions.SpawnAction; import com.google.devtools.build.lib.analysis.config.CoreOptionConverters.StrictDepsMode; import com.google.devtools.build.lib.analysis.starlark.StarlarkActionFactory; import com.google.devtools.build.lib.analysis.starlark.StarlarkRuleContext; -import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; import com.google.devtools.build.lib.collect.nestedset.Order; import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; import com.google.devtools.build.lib.rules.cpp.CcInfo; import com.google.devtools.build.lib.shell.ShellUtils; -import com.google.devtools.build.lib.vfs.FileSystemUtils; import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -315,37 +310,6 @@ private static List tokenize(List input) throws EvalException { return output; } - public Artifact stampJar( - StarlarkActionFactory actions, - Artifact inputJar, - Label targetLabel, - JavaToolchainProvider javaToolchain, - String execGroup) - throws EvalException { - String basename = FileSystemUtils.removeExtension(inputJar.getFilename()) + "-stamped.jar"; - Artifact outputJar = actions.declareFile(basename, inputJar); - // ijar doubles as a stamping tool - FilesToRunProvider ijarTarget = javaToolchain.getIjar(); - CustomCommandLine.Builder commandLine = - CustomCommandLine.builder() - .addExecPath(inputJar) - .addExecPath(outputJar) - .add("--nostrip_jar") - .addLabel("--target_label", targetLabel); - SpawnAction.Builder actionBuilder = - new SpawnAction.Builder() - .addInput(inputJar) - .addOutput(outputJar) - .setExecutable(ijarTarget) - .setProgressMessage("Stamping target label into jar %s", inputJar.getFilename()) - .addCommandLine(commandLine.build()) - .useDefaultShellEnvironment() - .setMnemonic("JavaIjar") - .setExecGroup(execGroup); - actions.registerAction(actionBuilder.build(actions.getActionConstructionContext())); - return outputJar; - } - private static StrictDepsMode getStrictDepsMode(String strictDepsMode) { switch (strictDepsMode) { case "OFF": diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java index e011eb3df3cf71..9dd07f9a19022d 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java @@ -213,22 +213,6 @@ private String getExecGroup(boolean useAutoExecGroups) { } } - @Override - public Artifact stampJar( - StarlarkActionFactory actions, - Artifact jar, - Label targetLabel, - JavaToolchainProvider javaToolchain) - throws EvalException { - return JavaInfoBuildHelper.getInstance() - .stampJar( - actions, - jar, - targetLabel, - javaToolchain, - getExecGroup(actions.getRuleContext().useAutoExecGroups())); - } - @Override public Artifact packSources( StarlarkActionFactory actions, diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java index 7e0df85cfb5560..e20653f6f1e5ae 100644 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java @@ -318,41 +318,6 @@ JavaInfoT createJavaCompileAction( StarlarkThread thread) throws EvalException, InterruptedException, RuleErrorException; - @StarlarkMethod( - name = "stamp_jar", - doc = - "Stamps a jar with a target label for add_dep support. " - + "The return value is typically passed to " - + "" - + "JavaInfo#compile_jar. " - + "Prefer to use " - + "run_ijar " - + "when possible.", - parameters = { - @Param(name = "actions", named = true, doc = "ctx.actions"), - @Param( - name = "jar", - positional = false, - named = true, - doc = "The jar to run stamp_jar on."), - @Param( - name = "target_label", - positional = false, - named = true, - doc = - "A target label to stamp the jar with. Used for add_dep support. " - + "Typically, you would pass ctx.label to stamp the jar " - + "with the current rule's label."), - @Param( - name = "java_toolchain", - positional = false, - named = true, - doc = "A JavaToolchainInfo to used to find the stamp_jar tool."), - }) - FileApi stampJar( - StarlarkActionFactoryT actions, FileT jar, Label targetLabel, JavaToolchainT javaToolchain) - throws EvalException; - @StarlarkMethod( name = "pack_sources", doc = diff --git a/src/main/starlark/builtins_bzl/common/java/java_common.bzl b/src/main/starlark/builtins_bzl/common/java/java_common.bzl index 21b9ecd03eb06e..91490fff8c68e5 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_common.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_common.bzl @@ -18,6 +18,7 @@ load(":common/java/java_info.bzl", "JavaInfo") load(":common/java/java_common_internal_for_builtins.bzl", "compile", "merge", "run_ijar") load(":common/java/java_plugin_info.bzl", "JavaPluginInfo") load(":common/java/java_semantics.bzl", "semantics") +load(":common/java/java_helper.bzl", "helper") _java_common_internal = _builtins.internal.java_common_internal_do_not_use @@ -96,12 +97,23 @@ def _stamp_jar(actions, jar, java_toolchain, target_label): (File) The output artifact """ - return _java_common_internal.stamp_jar( - actions = actions, - jar = jar, - java_toolchain = java_toolchain, - target_label = target_label, + output = actions.declare_file(helper.basename_without_extension(jar) + "-stamped.jar", sibling = jar) + args = actions.args() + args.add(jar) + args.add(output) + args.add("--nostrip_jar") + args.add("--target_label", target_label) + actions.run( + mnemonic = "JavaIjar", + inputs = [jar], + outputs = [output], + executable = java_toolchain.ijar, # ijar doubles as a stamping tool + arguments = [args], + progress_message = "Stamping target label into jar %{input}", + toolchain = semantics.JAVA_TOOLCHAIN_TYPE, + use_default_shell_env = True, ) + return output def _pack_sources( actions, diff --git a/src/test/java/com/google/devtools/build/lib/analysis/AutoExecGroupsTest.java b/src/test/java/com/google/devtools/build/lib/analysis/AutoExecGroupsTest.java index 87b41522cc7c4b..fd186783b4f5fd 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/AutoExecGroupsTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/AutoExecGroupsTest.java @@ -1570,11 +1570,15 @@ public void javaCommonStampJar_automaticExecGroupsEnabled_actionExecutesOnFirstP "custom_rule(name = 'custom_rule_name')"); useConfiguration("--incompatible_auto_exec_groups"); - ImmutableList actions = getActions("//test:custom_rule_name", SpawnAction.class); + ImmutableList actions = + getActions("//test:custom_rule_name").stream() + .filter(action -> action.getMnemonic().equals("JavaIjar")) + .collect(toImmutableList()); + ; assertThat(actions).hasSize(1); assertThat(actions.get(0).getProgressMessage()) - .isEqualTo("Stamping target label into jar lib_custom_rule_name.jar"); + .matches("Stamping target label into jar .*/lib_custom_rule_name.jar"); assertThat(actions.get(0).getOwner().getExecutionPlatform().label()) .isEqualTo(Label.parseCanonical("//platforms:platform_1")); } From 98dd941e1bf1a6db01e5cf603e82435d7b746707 Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 02:45:48 -0700 Subject: [PATCH 25/51] Fix translation of Starlark `JavaInfo` into native Once more `java_common` APIs move to Starlark, JavaInfo instances will no longer only be constructed in Starlark via the `init` constructor. So the internal provider fields may have been constructed both in native or Starlark. This change updates the translation to handle both cases. As a happy side-effect, it also gets rid of the private "_plugins" field and instead reconstructs `JavaPluginInfo` from the public API methods. PiperOrigin-RevId: 545611155 Change-Id: Id3d287c4966804a6fb21aa2c661935f152aaa0d6 --- .../lib/rules/java/BootClassPathInfo.java | 21 +++ .../java/JavaCompilationInfoProvider.java | 45 ++++++ .../build/lib/rules/java/JavaInfo.java | 2 +- .../rules/java/JavaModuleFlagsProvider.java | 34 ++++- .../build/lib/rules/java/JavaPluginInfo.java | 20 ++- .../java/JavaRuleOutputJarsProvider.java | 26 +++- .../builtins_bzl/common/java/java_info.bzl | 2 - .../devtools/build/lib/rules/java/BUILD | 3 + .../rules/java/JavaInfoStarlarkApiTest.java | 131 ++++++++++++++++++ 9 files changed, 267 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/BootClassPathInfo.java b/src/main/java/com/google/devtools/build/lib/rules/java/BootClassPathInfo.java index ad5f49b1a63e1b..4c0bc096ace14d 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/BootClassPathInfo.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/BootClassPathInfo.java @@ -23,6 +23,7 @@ import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; import com.google.devtools.build.lib.packages.BuiltinProvider; import com.google.devtools.build.lib.packages.NativeInfo; +import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; import com.google.devtools.build.lib.starlarkbuildapi.FileApi; import com.google.devtools.build.lib.starlarkbuildapi.core.ProviderApi; import com.google.devtools.build.lib.vfs.PathFragment; @@ -46,6 +47,26 @@ public final class BootClassPathInfo extends NativeInfo implements StarlarkValue /** Provider singleton constant. */ public static final Provider PROVIDER = new Provider(); + /** + * Transforms a {@link BootClassPathInfo} instance received from Starlark to native. + * + *

For now, only native instances exist, so we handle the null/None case and cast otherwise. + * + * @param obj the instance to translate + * @return a {@link BootClassPathInfo} provider instance + * @throws RuleErrorException if the provided instance is of an incompatible type + */ + // TODO(hvd): In the future, when this provider is moved to Starlark, this method must be + // updated to translate the StarlarkInfo. + static BootClassPathInfo fromStarlark(Object obj) throws RuleErrorException { + if (obj == null || obj == Starlark.NONE) { + return empty(); + } else if (obj instanceof BootClassPathInfo) { + return (BootClassPathInfo) obj; + } + throw new RuleErrorException("expected BootClassPathInfo, got: " + Starlark.type(obj)); + } + /** Provider class for {@link BootClassPathInfo} objects. */ @StarlarkBuiltin(name = "Provider", documented = false, doc = "") public static class Provider extends BuiltinProvider implements ProviderApi { diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationInfoProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationInfoProvider.java index 91154791aa1290..da766f8160fec7 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationInfoProvider.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompilationInfoProvider.java @@ -20,10 +20,15 @@ import com.google.devtools.build.lib.collect.nestedset.Depset; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; +import com.google.devtools.build.lib.packages.StructImpl; import com.google.devtools.build.lib.rules.java.JavaInfo.JavaInfoInternalProvider; import com.google.devtools.build.lib.starlarkbuildapi.java.JavaCompilationInfoProviderApi; import com.google.errorprone.annotations.CanIgnoreReturnValue; import javax.annotation.Nullable; +import net.starlark.java.eval.EvalException; +import net.starlark.java.eval.Sequence; +import net.starlark.java.eval.Starlark; /** * A class that provides compilation information in Java rules, for perusal of aspects and tools. @@ -36,6 +41,46 @@ public final class JavaCompilationInfoProvider @Nullable private final NestedSet compilationClasspath; private final BootClassPathInfo bootClasspath; + /** + * Transforms the {@code compilation_info} field from a {@link JavaInfo} into a native instance. + * + * @param javaInfo A {@link JavaInfo} instance. + * @return a {@link JavaCompilationInfoProvider} instance or {@code null} if the {@code + * compilation_info} field is not present in the supplied {@code javaInfo} + * @throws RuleErrorException if the {@code compilation_info} is of an incompatible type + * @throws EvalException if there are any errors accessing Starlark values + */ + @Nullable + static JavaCompilationInfoProvider fromStarlarkJavaInfo(StructImpl javaInfo) + throws RuleErrorException, EvalException { + Object value = javaInfo.getValue("compilation_info"); + if (value == null || value == Starlark.NONE) { + return null; + } else if (value instanceof JavaCompilationInfoProvider) { + return (JavaCompilationInfoProvider) value; + } else if (value instanceof StructImpl) { + StructImpl info = (StructImpl) value; + Builder builder = + new Builder() + .setJavacOpts( + Sequence.cast(info.getValue("javac_options"), String.class, "javac_options") + .getImmutableList()) + .setBootClasspath(BootClassPathInfo.fromStarlark(info.getValue("boot_classpath"))); + Object runtimeClasspath = info.getValue("runtime_classpath"); + if (runtimeClasspath != null) { + builder.setRuntimeClasspath( + Depset.noneableCast(runtimeClasspath, Artifact.class, "runtime_classpath")); + } + Object compilationClasspath = info.getValue("compilation_classpath"); + if (compilationClasspath != null) { + builder.setCompilationClasspath( + Depset.noneableCast(compilationClasspath, Artifact.class, "compilation_classpath")); + } + return builder.build(); + } + throw new RuleErrorException("expected java_compilation_info, got: " + Starlark.type(value)); + } + @Override public boolean isImmutable() { return true; // immutable and Starlark-hashable diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfo.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfo.java index 32e23782660e62..fd612854c6902d 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfo.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfo.java @@ -396,7 +396,7 @@ private JavaInfo(StructImpl javaInfo) throws EvalException, TypeException, RuleE this( JavaCcInfoProvider.fromStarlarkJavaInfo(javaInfo), JavaCompilationArgsProvider.fromStarlarkJavaInfo(javaInfo), - /* javaCompilationInfoProvider= */ null, + JavaCompilationInfoProvider.fromStarlarkJavaInfo(javaInfo), JavaGenJarsProvider.from(javaInfo.getValue("annotation_processing")), JavaModuleFlagsProvider.fromStarlarkJavaInfo(javaInfo), JavaPluginInfo.fromStarlarkJavaInfo(javaInfo), diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaModuleFlagsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaModuleFlagsProvider.java index 66dbf52d910248..3867c64e30286f 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaModuleFlagsProvider.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaModuleFlagsProvider.java @@ -26,13 +26,16 @@ import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; import com.google.devtools.build.lib.collect.nestedset.Order; import com.google.devtools.build.lib.packages.AttributeMap; +import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; import com.google.devtools.build.lib.packages.StructImpl; import com.google.devtools.build.lib.rules.java.JavaInfo.JavaInfoInternalProvider; import com.google.devtools.build.lib.starlarkbuildapi.java.JavaModuleFlagsProviderApi; import java.util.Collection; import java.util.List; import java.util.stream.Stream; +import javax.annotation.Nullable; import net.starlark.java.eval.EvalException; +import net.starlark.java.eval.Starlark; /** * Provides information about {@code --add-exports=} and {@code --add-opens=} flags for Java @@ -113,12 +116,31 @@ public ImmutableList toFlags() { return toFlags(addExports().toList(), addOpens().toList()); } + /** + * Translates the {@code module_flags_info} from a {@link JavaInfo} to the native class. + * + * @param javaInfo a {@link JavaInfo} provider instance + * @return a {@link JavaModuleFlagsProvider} instance or {@code null} if {@code module_flags_info} + * is absent or {@code None} + * @throws EvalException if there are any errors accessing Starlark values + * @throws TypeException if any depset values are of an incompatible type + * @throws RuleErrorException if the {@code module_flags_info} is of an incompatible type + */ + @Nullable static JavaModuleFlagsProvider fromStarlarkJavaInfo(StructImpl javaInfo) - throws EvalException, TypeException { - StructImpl moduleFlagsInfo = javaInfo.getValue("module_flags_info", StructImpl.class); - return JavaModuleFlagsProvider.create( - moduleFlagsInfo.getValue("add_exports", Depset.class).toList(String.class), - moduleFlagsInfo.getValue("add_opens", Depset.class).toList(String.class), - Stream.empty()); + throws EvalException, TypeException, RuleErrorException { + Object value = javaInfo.getValue("module_flags_info"); + if (value == null || value == Starlark.NONE) { + return null; + } else if (value instanceof JavaModuleFlagsProvider) { + return (JavaModuleFlagsProvider) value; + } else if (value instanceof StructImpl) { + StructImpl moduleFlagsInfo = (StructImpl) value; + return JavaModuleFlagsProvider.create( + moduleFlagsInfo.getValue("add_exports", Depset.class).toList(String.class), + moduleFlagsInfo.getValue("add_opens", Depset.class).toList(String.class), + Stream.empty()); + } + throw new RuleErrorException("expected JavaModuleFlagsInfo, got: " + Starlark.type(value)); } } diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaPluginInfo.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPluginInfo.java index ba058ef2965356..02f0f3af6245bb 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaPluginInfo.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPluginInfo.java @@ -34,7 +34,6 @@ import com.google.devtools.build.lib.starlarkbuildapi.java.JavaPluginInfoApi; import java.util.ArrayList; import java.util.List; -import javax.annotation.Nullable; import net.starlark.java.eval.EvalException; import net.starlark.java.eval.Sequence; import net.starlark.java.eval.Starlark; @@ -318,10 +317,23 @@ public JavaPluginInfo disableAnnotationProcessing() { plugins().disableAnnotationProcessing(), /* generatesApi= */ false, getJavaOutputs()); } - @Nullable + /** + * Translates the plugin information from a {@link JavaInfo} instance. + * + * @param javaInfo the {@link JavaInfo} instance + * @return a {@link JavaPluginInfo} instance + * @throws EvalException if there are any errors accessing Starlark values + * @throws RuleErrorException if the {@code plugins} or {@code api_generating_plugins} fields are + * of an incompatible type + */ static JavaPluginInfo fromStarlarkJavaInfo(StructImpl javaInfo) throws EvalException, RuleErrorException { - Info info = javaInfo.getValue("_plugin_info", Info.class); - return info == null ? null : JavaPluginInfo.PROVIDER.wrap(info); + JavaPluginData plugins = JavaPluginData.wrap(javaInfo.getValue("plugins")); + JavaPluginData apiGeneratingPlugins = + JavaPluginData.wrap(javaInfo.getValue("api_generating_plugins")); + if (plugins.isEmpty() && apiGeneratingPlugins.isEmpty()) { + return JavaPluginInfo.empty(); + } + return new AutoValue_JavaPluginInfo(ImmutableList.of(), plugins, apiGeneratingPlugins); } } diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaRuleOutputJarsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaRuleOutputJarsProvider.java index 6a3e58cf6a9812..6c17609d844ca1 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaRuleOutputJarsProvider.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaRuleOutputJarsProvider.java @@ -23,6 +23,7 @@ import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.collect.compacthashset.CompactHashSet; import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; import com.google.devtools.build.lib.packages.StructImpl; import com.google.devtools.build.lib.rules.java.JavaInfo.JavaInfoInternalProvider; import com.google.devtools.build.lib.rules.java.JavaRuleOutputJarsProvider.JavaOutput; @@ -36,6 +37,7 @@ import javax.annotation.Nullable; import net.starlark.java.eval.EvalException; import net.starlark.java.eval.Sequence; +import net.starlark.java.eval.Starlark; import net.starlark.java.eval.StarlarkList; /** Provides information about jar files produced by a Java rule. */ @@ -305,11 +307,27 @@ public JavaRuleOutputJarsProvider build() { } } - static JavaRuleOutputJarsProvider fromStarlarkJavaInfo(StructImpl javaInfo) throws EvalException { - Sequence outputs = javaInfo.getValue("java_outputs", StarlarkList.class); + /** + * Translates the {@code outputs} field of a {@link JavaInfo} instance into a native {@link + * JavaRuleOutputJarsProvider} instance. + * + * @param javaInfo the {@link JavaInfo} instance + * @return a {@link JavaRuleOutputJarsProvider} instance + * @throws EvalException if there are any errors accessing Starlark values + * @throws RuleErrorException if any of the {@code output} instances are of incompatible type + */ + static JavaRuleOutputJarsProvider fromStarlarkJavaInfo(StructImpl javaInfo) + throws EvalException, RuleErrorException { JavaRuleOutputJarsProvider.Builder builder = JavaRuleOutputJarsProvider.builder(); - for (StructImpl output : Sequence.cast(outputs, StructImpl.class, "outputs")) { - builder.addJavaOutput(JavaOutput.fromStarlarkJavaOutput(output)); + for (Object output : + Sequence.cast(javaInfo.getValue("java_outputs"), Object.class, "outputs")) { + if (output instanceof JavaOutput) { + builder.addJavaOutput((JavaOutput) output); + } else if (output instanceof StructImpl) { + builder.addJavaOutput(JavaOutput.fromStarlarkJavaOutput((StructImpl) output)); + } else { + throw new RuleErrorException("expected JavaOutput, got: " + Starlark.type(output)); + } } return builder.build(); } diff --git a/src/main/starlark/builtins_bzl/common/java/java_info.bzl b/src/main/starlark/builtins_bzl/common/java/java_info.bzl index bcfed54c7e764b..b5c091effc494a 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_info.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_info.bzl @@ -234,7 +234,6 @@ def _javainfo_init( direct = [output_jar], transitive = [dep._transitive_full_compile_time_jars for dep in exports + deps], ), - "_plugin_info": plugin_info, "_compile_time_java_dependencies": depset( order = "preorder", transitive = [dep._compile_time_java_dependencies for dep in exports] + @@ -310,7 +309,6 @@ JavaInfo, _new_javainfo = provider( "_transitive_full_compile_time_jars": "internal API, do not use", "_compile_time_java_dependencies": "internal API, do not use", "_neverlink": "internal API, do not use", - "_plugin_info": "internal API, do not use", "_constraints": "internal API, do not use", }, init = _javainfo_init, diff --git a/src/test/java/com/google/devtools/build/lib/rules/java/BUILD b/src/test/java/com/google/devtools/build/lib/rules/java/BUILD index 3c511fb6c17e5a..b307d706780009 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/java/BUILD +++ b/src/test/java/com/google/devtools/build/lib/rules/java/BUILD @@ -79,12 +79,15 @@ java_test( srcs = ["JavaInfoStarlarkApiTest.java"], deps = [ ":java_toolchain_test_util", + "//src/main/java/com/google/devtools/build/lib/actions:artifacts", "//src/main/java/com/google/devtools/build/lib/analysis:configured_target", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/collect/nestedset", "//src/main/java/com/google/devtools/build/lib/packages", "//src/main/java/com/google/devtools/build/lib/rules/cpp", "//src/main/java/com/google/devtools/build/lib/rules/java:java-compilation", + "//src/main/java/com/google/devtools/build/lib/vfs", + "//src/main/java/net/starlark/java/eval", "//src/test/java/com/google/devtools/build/lib/actions/util", "//src/test/java/com/google/devtools/build/lib/analysis/util", "//src/test/java/com/google/devtools/build/lib/testutil:TestConstants", diff --git a/src/test/java/com/google/devtools/build/lib/rules/java/JavaInfoStarlarkApiTest.java b/src/test/java/com/google/devtools/build/lib/rules/java/JavaInfoStarlarkApiTest.java index 676d17eee75654..a9befae845ec5b 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/java/JavaInfoStarlarkApiTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/java/JavaInfoStarlarkApiTest.java @@ -19,17 +19,32 @@ import static com.google.devtools.build.lib.actions.util.ActionsTestUtil.prettyArtifactNames; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactRoot; +import com.google.devtools.build.lib.actions.ArtifactRoot.RootType; +import com.google.devtools.build.lib.actions.util.ActionsTestUtil; import com.google.devtools.build.lib.analysis.ConfiguredTarget; import com.google.devtools.build.lib.analysis.util.BuildViewTestCase; import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.collect.nestedset.Depset; import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; import com.google.devtools.build.lib.packages.Info; +import com.google.devtools.build.lib.packages.StarlarkInfo; import com.google.devtools.build.lib.packages.StarlarkProvider; import com.google.devtools.build.lib.packages.StructImpl; +import com.google.devtools.build.lib.packages.StructProvider; import com.google.devtools.build.lib.rules.cpp.LibraryToLink; +import com.google.devtools.build.lib.rules.java.JavaPluginInfo.JavaPluginData; import com.google.devtools.build.lib.rules.java.JavaRuleOutputJarsProvider.JavaOutput; import com.google.devtools.build.lib.testutil.TestConstants; +import com.google.devtools.build.lib.vfs.Path; import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.io.IOException; +import java.util.Map; +import net.starlark.java.eval.StarlarkList; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -857,6 +872,122 @@ public void buildHelperCreateJavaInfoWithModuleFlags() throws Exception { .containsExactly("--add-opens=java.base/java.lang=ALL-UNNAMED"); } + @Test + public void translateStarlarkJavaInfo_minimal() throws Exception { + ImmutableMap fields = getBuilderWithMandataryFields().buildOrThrow(); + StarlarkInfo starlarkInfo = makeStruct(fields); + + JavaInfo javaInfo = JavaInfo.PROVIDER.wrap(starlarkInfo); + + assertThat(javaInfo).isNotNull(); + assertThat(javaInfo.getCompilationInfoProvider()).isNull(); + assertThat(javaInfo.getJavaModuleFlagsInfo()).isEqualTo(JavaModuleFlagsProvider.EMPTY); + assertThat(javaInfo.getJavaPluginInfo()).isEqualTo(JavaPluginInfo.empty()); + } + + @Test + public void translateStarlarkJavaInfo_compilationInfo() throws Exception { + ImmutableMap fields = + getBuilderWithMandataryFields() + .put( + "compilation_info", + makeStruct( + ImmutableMap.of( + "javac_options", StarlarkList.immutableOf("opt1", "opt2"), + "boot_classpath", + BootClassPathInfo.create( + NestedSetBuilder.create( + Order.STABLE_ORDER, createArtifact("cp.jar")))))) + .buildOrThrow(); + StarlarkInfo starlarkInfo = makeStruct(fields); + + JavaInfo javaInfo = JavaInfo.PROVIDER.wrap(starlarkInfo); + + assertThat(javaInfo).isNotNull(); + assertThat(javaInfo.getCompilationInfoProvider()).isNotNull(); + assertThat(javaInfo.getCompilationInfoProvider().getJavacOpts()) + .containsExactly("opt1", "opt2"); + assertThat(javaInfo.getCompilationInfoProvider().getBootClasspath()).hasSize(1); + assertThat(prettyArtifactNames(javaInfo.getCompilationInfoProvider().getBootClasspath())) + .containsExactly("cp.jar"); + } + + @Test + public void translateStarlarkJavaInfo_moduleFlagsInfo() throws Exception { + ImmutableMap fields = + getBuilderWithMandataryFields() + .put( + "module_flags_info", + makeStruct( + ImmutableMap.of( + "add_exports", makeDepset(String.class, "export1", "export2"), + "add_opens", makeDepset(String.class, "open1", "open2")))) + .buildOrThrow(); + StarlarkInfo starlarkInfo = makeStruct(fields); + + JavaInfo javaInfo = JavaInfo.PROVIDER.wrap(starlarkInfo); + + assertThat(javaInfo).isNotNull(); + assertThat(javaInfo.getJavaModuleFlagsInfo()).isNotNull(); + assertThat(javaInfo.getJavaModuleFlagsInfo().getAddExports().toList()) + .containsExactly("export1", "export2"); + assertThat(javaInfo.getJavaModuleFlagsInfo().getAddOpens().toList()) + .containsExactly("open1", "open2"); + } + + @Test + public void translateStarlarkJavaInfo_pluginInfo() throws Exception { + ImmutableMap fields = + getBuilderWithMandataryFields() + .put( + "plugins", + JavaPluginData.create( + NestedSetBuilder.create(Order.STABLE_ORDER, "c1", "c2", "c3"), + NestedSetBuilder.create(Order.STABLE_ORDER, createArtifact("f1")), + NestedSetBuilder.emptySet(Order.STABLE_ORDER))) + .buildKeepingLast(); + StarlarkInfo starlarkInfo = makeStruct(fields); + + JavaInfo javaInfo = JavaInfo.PROVIDER.wrap(starlarkInfo); + + assertThat(javaInfo).isNotNull(); + assertThat(javaInfo.plugins()).isNotNull(); + assertThat(javaInfo.plugins().processorClasses().toList()).containsExactly("c1", "c2", "c3"); + assertThat(prettyArtifactNames(javaInfo.plugins().processorClasspath())).containsExactly("f1"); + } + + private static ImmutableMap.Builder getBuilderWithMandataryFields() { + Depset emptyDepset = Depset.of(Artifact.class, NestedSetBuilder.create(Order.STABLE_ORDER)); + return ImmutableMap.builder() + .put("transitive_native_libraries", emptyDepset) + .put("compile_jars", emptyDepset) + .put("full_compile_jars", emptyDepset) + .put("transitive_compile_time_jars", emptyDepset) + .put("transitive_runtime_jars", emptyDepset) + .put("_transitive_full_compile_time_jars", emptyDepset) + .put("_compile_time_java_dependencies", emptyDepset) + .put("plugins", JavaPluginData.empty()) + .put("api_generating_plugins", JavaPluginData.empty()) + .put("java_outputs", StarlarkList.empty()) + .put("transitive_source_jars", emptyDepset) + .put("source_jars", StarlarkList.empty()) + .put("runtime_output_jars", StarlarkList.empty()); + } + + private Artifact createArtifact(String path) throws IOException { + Path execRoot = scratch.dir("/"); + ArtifactRoot root = ArtifactRoot.asDerivedRoot(execRoot, RootType.Output, "fake-root"); + return ActionsTestUtil.createArtifact(root, path); + } + + private static Depset makeDepset(Class clazz, T... elems) { + return Depset.of(clazz, NestedSetBuilder.create(Order.STABLE_ORDER, elems)); + } + + private static StarlarkInfo makeStruct(Map struct) { + return StructProvider.STRUCT.create(struct, ""); + } + private RuleBuilder ruleBuilder() { return new RuleBuilder(); } From fc5eb406fc8c7b036f34a035724351c0b9f1e01e Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 03:52:22 -0700 Subject: [PATCH 26/51] Delete dead code PiperOrigin-RevId: 545623344 Change-Id: I7fb4bf7c35ec8f3cc965820236b8d7a00272389e --- .../rules/java/SingleJarActionBuilder.java | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/SingleJarActionBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/java/SingleJarActionBuilder.java index 04b4f104330d12..1b335cf4f5198d 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/SingleJarActionBuilder.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/SingleJarActionBuilder.java @@ -29,8 +29,6 @@ import com.google.devtools.build.lib.analysis.actions.CustomCommandLine.VectorArg; import com.google.devtools.build.lib.analysis.actions.SpawnAction; import com.google.devtools.build.lib.collect.nestedset.NestedSet; -import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; -import com.google.devtools.build.lib.collect.nestedset.Order; import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; import java.util.function.Consumer; @@ -111,34 +109,6 @@ public static void createSourceJarAction( actionRegistry.registerAction(builder.build(actionConstructionContext)); } - /** - * Creates an Action that merges jars into a single archive. - * - * @param jars the jars to merge. - * @param output the output jar to create - */ - public static void createSingleJarAction( - RuleContext ruleContext, NestedSet jars, Artifact output) { - requireNonNull(ruleContext); - requireNonNull(jars); - requireNonNull(output); - SpawnAction.Builder builder = - new SpawnAction.Builder() - .setExecutable(JavaToolchainProvider.from(ruleContext).getSingleJar()) - .addOutput(output) - .addTransitiveInputs(jars) - .addCommandLine( - sourceJarCommandLine( - output, - /* semantics= */ null, - /* resources= */ NestedSetBuilder.emptySet(Order.STABLE_ORDER), - jars), - ParamFileInfo.builder(ParameterFileType.SHELL_QUOTED).setUseAlways(true).build()) - .setProgressMessage("Building singlejar jar %{output}") - .setMnemonic("JavaSingleJar"); - ruleContext.registerAction(builder.build(ruleContext)); - } - private static CommandLine sourceJarCommandLine( Artifact outputJar, JavaSemantics semantics, From 53ca28cfaaaf66feb135c719f56e98d7dc31cbf6 Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 05:04:46 -0700 Subject: [PATCH 27/51] Removing aquery's `--deduplicate_depsets`. It's not used explicitly anywhere and is true by default anyway. PiperOrigin-RevId: 545636451 Change-Id: I1ae898f1884774532ded7697e97869ab883f0640 --- .../devtools/build/lib/buildtool/AqueryProcessor.java | 1 - .../devtools/build/lib/buildtool/BuildTool.java | 1 - .../ActionGraphProtoOutputFormatterCallback.java | 1 - .../build/lib/query2/aquery/AqueryOptions.java | 11 ----------- .../lib/skyframe/actiongraph/v2/ActionGraphDump.java | 5 +---- .../lib/skyframe/actiongraph/v2/KnownNestedSets.java | 9 ++------- 6 files changed, 3 insertions(+), 25 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/AqueryProcessor.java b/src/main/java/com/google/devtools/build/lib/buildtool/AqueryProcessor.java index f804dd30db86ca..ca06040c1ae787 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/AqueryProcessor.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/AqueryProcessor.java @@ -87,7 +87,6 @@ public BlazeCommandResult dumpActionGraphFromSkyframe(CommandEnvironment env) { aqueryOptions.includeArtifacts, actionFilters, aqueryOptions.includeParamFiles, - aqueryOptions.deduplicateDepsets, aqueryOptions.includeFileWriteContents, aqueryOutputHandler, env.getReporter()); diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java index bd41e5278432a5..b976d1f41cc875 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java @@ -434,7 +434,6 @@ private void dumpSkyframeStateAfterBuild( /* includeArtifacts= */ true, /* actionFilters= */ null, /* includeParamFiles= */ false, - /* deduplicateDepsets= */ true, /* includeFileWriteContents */ false, aqueryOutputHandler, getReporter()); diff --git a/src/main/java/com/google/devtools/build/lib/query2/aquery/ActionGraphProtoOutputFormatterCallback.java b/src/main/java/com/google/devtools/build/lib/query2/aquery/ActionGraphProtoOutputFormatterCallback.java index 5cbaf4846d321f..0de141643d5d4a 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/aquery/ActionGraphProtoOutputFormatterCallback.java +++ b/src/main/java/com/google/devtools/build/lib/query2/aquery/ActionGraphProtoOutputFormatterCallback.java @@ -74,7 +74,6 @@ public class ActionGraphProtoOutputFormatterCallback extends AqueryThreadsafeCal options.includeArtifacts, this.actionFilters, options.includeParamFiles, - options.deduplicateDepsets, options.includeFileWriteContents, aqueryOutputHandler, eventHandler); diff --git a/src/main/java/com/google/devtools/build/lib/query2/aquery/AqueryOptions.java b/src/main/java/com/google/devtools/build/lib/query2/aquery/AqueryOptions.java index 81071863f75873..6deee8958f623e 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/aquery/AqueryOptions.java +++ b/src/main/java/com/google/devtools/build/lib/query2/aquery/AqueryOptions.java @@ -78,15 +78,4 @@ public class AqueryOptions extends CommonQueryOptions { + "Note: Specifying a target with --skyframe_state is currently not supported. " + "This flag is only available with --output=proto or --output=textproto.") public boolean queryCurrentSkyframeState; - - @Option( - name = "deduplicate_depsets", - defaultValue = "true", - documentationCategory = OptionDocumentationCategory.QUERY, - effectTags = {OptionEffectTag.TERMINAL_OUTPUT}, - help = - "De-duplicate non-leaf children of a dep_set_of_files in the final proto/textproto/json" - + " output. This does not deduplicate depsets that don't share an immediate parent." - + " This does not affect the final effective list of input artifacts of the actions.") - public boolean deduplicateDepsets; } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/ActionGraphDump.java b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/ActionGraphDump.java index 2dcb8728a33c13..7e80cc27864339 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/ActionGraphDump.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/ActionGraphDump.java @@ -79,7 +79,6 @@ public ActionGraphDump( boolean includeArtifacts, AqueryActionFilter actionFilters, boolean includeParamFiles, - boolean deduplicateDepsets, boolean includeFileWriteContents, AqueryOutputHandler aqueryOutputHandler, ExtendedEventHandler eventHandler) { @@ -89,7 +88,6 @@ public ActionGraphDump( includeArtifacts, actionFilters, includeParamFiles, - deduplicateDepsets, includeFileWriteContents, aqueryOutputHandler, eventHandler); @@ -101,7 +99,6 @@ public ActionGraphDump( boolean includeArtifacts, AqueryActionFilter actionFilters, boolean includeParamFiles, - boolean deduplicateDepsets, boolean includeFileWriteContents, AqueryOutputHandler aqueryOutputHandler, ExtendedEventHandler eventHandler) { @@ -117,7 +114,7 @@ public ActionGraphDump( KnownRuleClassStrings knownRuleClassStrings = new KnownRuleClassStrings(aqueryOutputHandler); knownArtifacts = new KnownArtifacts(aqueryOutputHandler); knownConfigurations = new KnownConfigurations(aqueryOutputHandler); - knownNestedSets = new KnownNestedSets(aqueryOutputHandler, knownArtifacts, deduplicateDepsets); + knownNestedSets = new KnownNestedSets(aqueryOutputHandler, knownArtifacts); knownAspectDescriptors = new KnownAspectDescriptors(aqueryOutputHandler); knownTargets = new KnownTargets(aqueryOutputHandler, knownRuleClassStrings); } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownNestedSets.java b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownNestedSets.java index a3a759d53b66af..25e31401881064 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownNestedSets.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/KnownNestedSets.java @@ -24,15 +24,10 @@ /** Cache for NestedSets in the action graph. */ public class KnownNestedSets extends BaseCache { private final KnownArtifacts knownArtifacts; - private final boolean deduplicateDepsets; - KnownNestedSets( - AqueryOutputHandler aqueryOutputHandler, - KnownArtifacts knownArtifacts, - boolean deduplicateDepsets) { + KnownNestedSets(AqueryOutputHandler aqueryOutputHandler, KnownArtifacts knownArtifacts) { super(aqueryOutputHandler); this.knownArtifacts = knownArtifacts; - this.deduplicateDepsets = deduplicateDepsets; } @Override @@ -50,7 +45,7 @@ DepSetOfFiles createProto(Object nestedSetObject, int id) // meaningful info and sometimes even corrupt the proto3 output. More context: b/186193294. Set visited = new HashSet<>(); for (NestedSet succ : nestedSet.getNonLeaves()) { - if (!deduplicateDepsets || visited.add(succ.toNode())) { + if (visited.add(succ.toNode())) { depSetBuilder.addTransitiveDepSetIds(this.dataToIdAndStreamOutputProto(succ)); } } From fb564ea90809d88db57e9992f973c14dd0d17da9 Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 05:33:52 -0700 Subject: [PATCH 28/51] Move deprecated fileset options to the graveyard. PiperOrigin-RevId: 545641304 Change-Id: I21d5846e06e6f45b55b0ab883ad5ad7e8f76a846 --- .../lib/analysis/config/CoreOptions.java | 11 +--------- .../lib/bazel/rules/BazelRulesModule.java | 20 +++++++++++++------ 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java b/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java index 091aa46ac9f040..c5317f2a3199c4 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java @@ -173,8 +173,7 @@ public class CoreOptions extends FragmentOptions implements Cloneable { effectTags = {OptionEffectTag.BUILD_FILE_SEMANTICS, OptionEffectTag.EAGERNESS_TO_EXIT}, help = "If this option is enabled, filesets crossing package boundaries are reported " - + "as errors. It does not work when check_fileset_dependencies_recursively is " - + "disabled.") + + "as errors.") public boolean strictFilesets; @Option( @@ -499,14 +498,6 @@ public ExecConfigurationDistinguisherSchemeConverter() { + "https://bazel.build/extending/rules#runfiles_features_to_avoid).") public boolean alwaysIncludeFilesToBuildInData; - @Option( - name = "experimental_skyframe_native_filesets", - defaultValue = "true", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - effectTags = {OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION}, - deprecationWarning = "This flag is a no-op and skyframe-native-filesets is always true.") - public boolean skyframeNativeFileset; - @Option( name = "run_under", defaultValue = "null", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java index 04d3c0b3d2b057..8ed88b7b019213 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java @@ -467,6 +467,16 @@ public static class BuildGraveyardOptions extends OptionsBase { help = "This option is deprecated and has no effect and will be removed in the future.") public boolean deferParamFiles; + @Option( + name = "experimental_throttle_action_cache_check", + defaultValue = "true", + converter = BooleanConverter.class, + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + metadataTags = OptionMetadataTag.EXPERIMENTAL, + effectTags = {OptionEffectTag.EXECUTION}, + help = "no-op") + public boolean throttleActionCacheCheck; + @Option( name = "check_fileset_dependencies_recursively", defaultValue = "true", @@ -478,14 +488,12 @@ public static class BuildGraveyardOptions extends OptionsBase { public boolean checkFilesetDependenciesRecursively; @Option( - name = "experimental_throttle_action_cache_check", + name = "experimental_skyframe_native_filesets", defaultValue = "true", - converter = BooleanConverter.class, documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - metadataTags = OptionMetadataTag.EXPERIMENTAL, - effectTags = {OptionEffectTag.EXECUTION}, - help = "no-op") - public boolean throttleActionCacheCheck; + effectTags = {OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION}, + deprecationWarning = "This flag is a no-op and skyframe-native-filesets is always true.") + public boolean skyframeNativeFileset; } /** This is where deprecated Bazel-specific options only used by the build command go to die. */ From d426b3d36d6fd2e0427caa1a64bd629bfaab4569 Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 05:37:03 -0700 Subject: [PATCH 29/51] Do not invalidate skyframe's in-memory action cache when changing output mode. Previously, we invalidate skyframe's cache if the output mode is changed from BwoB to ALL because they were two different code paths and are incompatible with each other. Now the code paths are unifded and changing the output mode only affects the decision for whether Bazel should download outputs, no more other things. PiperOrigin-RevId: 545641839 Change-Id: I83288f7cb505ad0e47368c702797ca365bdb4191 --- .../build/lib/skyframe/SkyframeExecutor.java | 5 +---- .../BuildWithoutTheBytesIntegrationTest.java | 17 +++++++++++------ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java index 2e4ba6b327bf56..5a74f4933f205b 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java @@ -1722,11 +1722,8 @@ public void deleteActionsIfRemoteOptionsChanged(OptionsProvider options) lastRemoteCacheEnabled != null && lastRemoteCacheEnabled && !remoteCacheEnabled; } lastRemoteCacheEnabled = remoteCacheEnabled; - - RemoteOutputsMode remoteOutputsMode = + lastRemoteOutputsMode = remoteOptions != null ? remoteOptions.remoteOutputsMode : RemoteOutputsMode.ALL; - needsDeletion |= lastRemoteOutputsMode != null && lastRemoteOutputsMode != remoteOutputsMode; - this.lastRemoteOutputsMode = remoteOutputsMode; if (needsDeletion) { memoizingEvaluator.delete(k -> SkyFunctions.ACTION_EXECUTION.equals(k.functionName())); diff --git a/src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTest.java b/src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTest.java index 412bdd0bc90e21..b568707e64f280 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTest.java @@ -194,7 +194,7 @@ public void intermediateOutputsAreInputForInternalActions_prefetchIntermediateOu } @Test - public void changeOutputMode_invalidateActions() throws Exception { + public void changeOutputMode_notInvalidateActions() throws Exception { write( "a/BUILD", "genrule(", @@ -210,21 +210,26 @@ public void changeOutputMode_invalidateActions() throws Exception { " outs = ['foobar.txt'],", " cmd = 'cat $(location :foo) > $@ && echo bar > $@',", ")"); + // Download all outputs with regex so in the next build with ALL mode, the actions are not + // invalidated because of missing outputs. + addOptions("--experimental_remote_download_regex=.*"); ActionEventCollector actionEventCollector = new ActionEventCollector(); runtimeWrapper.registerSubscriber(actionEventCollector); buildTarget("//a:foobar"); + // Add the new option here because waitDownloads below will internally create a new command + // which will parse the new option. + setDownloadAll(); + waitDownloads(); // 3 = workspace status action + //:foo + //:foobar assertThat(actionEventCollector.getNumActionNodesEvaluated()).isEqualTo(3); actionEventCollector.clear(); events.clear(); - setDownloadAll(); buildTarget("//a:foobar"); - // Changing output mode should invalidate SkyFrame's in-memory caching and make it re-evaluate - // the action nodes. - assertThat(actionEventCollector.getNumActionNodesEvaluated()).isEqualTo(3); - events.assertContainsInfo("2 processes: 2 remote cache hit"); + // Changing output mode should not invalidate SkyFrame's in-memory caching. + assertThat(actionEventCollector.getNumActionNodesEvaluated()).isEqualTo(0); + events.assertContainsInfo("0 processes"); } @Test From 2bba03b1f6d446c63634ffa82e3e86bb94112816 Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 06:34:14 -0700 Subject: [PATCH 30/51] Use an ImmutableMap instead of a HashMap for retry times. PiperOrigin-RevId: 545652311 Change-Id: I37f455adfe5e29e2e0109856c004b7575a99fc05 --- .../devtools/build/lib/actions/SpawnMetrics.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/actions/SpawnMetrics.java b/src/main/java/com/google/devtools/build/lib/actions/SpawnMetrics.java index 0f7392cfe23cd3..3109e08e42025d 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/SpawnMetrics.java +++ b/src/main/java/com/google/devtools/build/lib/actions/SpawnMetrics.java @@ -15,6 +15,7 @@ import com.google.common.base.Joiner; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.util.ArrayList; import java.util.HashMap; @@ -69,8 +70,9 @@ public static SpawnMetrics forLocalExecution(int wallTimeInMs) { private final int executionWallTimeInMs; private final int processOutputsTimeInMs; private final int networkTimeInMs; + // error code to duration in ms - private final Map retryTimeInMs; + private final ImmutableMap retryTimeInMs; private final long inputBytes; private final long inputFiles; private final long memoryEstimateBytes; @@ -91,7 +93,7 @@ private SpawnMetrics(Builder builder) { this.setupTimeInMs = builder.setupTimeInMs; this.uploadTimeInMs = builder.uploadTimeInMs; this.executionWallTimeInMs = builder.executionWallTimeInMs; - this.retryTimeInMs = builder.retryTimeInMs; + this.retryTimeInMs = ImmutableMap.copyOf(builder.retryTimeInMs); this.processOutputsTimeInMs = builder.processOutputsTimeInMs; this.inputBytes = builder.inputBytes; this.inputFiles = builder.inputFiles; @@ -436,8 +438,8 @@ public Builder addRetryTimeInMs(int errorCode, int retryTimeInMs) { } @CanIgnoreReturnValue - public Builder setRetryTimeInMs(Map retryTimeInMs) { - this.retryTimeInMs = new HashMap<>(retryTimeInMs); + public Builder setRetryTimeInMs(ImmutableMap retryTimeInMs) { + this.retryTimeInMs = retryTimeInMs; return this; } From 24a0325428c25dfc632e4d0cfec53a651a3bb68e Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 06:49:11 -0700 Subject: [PATCH 31/51] Move no-op flag `experimental_skyframe_prepare_analysis` to graveyard. PiperOrigin-RevId: 545655302 Change-Id: I1f549708a532465ce2e89940ecfe3a57530a1974 --- .../devtools/build/lib/analysis/AnalysisOptions.java | 9 --------- .../devtools/build/lib/bazel/rules/BazelRulesModule.java | 9 +++++++++ .../devtools/build/lib/analysis/BuildViewTest.java | 9 --------- .../build/lib/analysis/util/AnalysisTestCase.java | 3 --- 4 files changed, 9 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisOptions.java b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisOptions.java index 7cd66de8b88283..5ccce705a0a798 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisOptions.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisOptions.java @@ -93,15 +93,6 @@ public class AnalysisOptions extends OptionsBase { ) public long versionWindowForDirtyNodeGc; - @Option( - name = "experimental_skyframe_prepare_analysis", - deprecationWarning = "This flag is a no-op and will be deleted in a future release.", - defaultValue = "false", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - effectTags = {OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION}, - help = "Deprecated. No-op.") - public boolean skyframePrepareAnalysis; - @Option( name = "experimental_skyframe_cpu_heavy_skykeys_thread_pool_size", defaultValue = "HOST_CPUS", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java index 8ed88b7b019213..0ce9cb9b903a53 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java @@ -44,6 +44,15 @@ public final class BazelRulesModule extends BlazeModule { */ @SuppressWarnings("deprecation") // These fields have no JavaDoc by design public static class BuildGraveyardOptions extends OptionsBase { + @Option( + name = "experimental_skyframe_prepare_analysis", + deprecationWarning = "This flag is a no-op and will be deleted in a future release.", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = {OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION}, + help = "Deprecated. No-op.") + public boolean skyframePrepareAnalysis; + @Option( name = "incompatible_use_platforms_repo_for_constraints", defaultValue = "true", diff --git a/src/test/java/com/google/devtools/build/lib/analysis/BuildViewTest.java b/src/test/java/com/google/devtools/build/lib/analysis/BuildViewTest.java index 28bdc4ffdd443d..049b1ed8f0f162 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/BuildViewTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/BuildViewTest.java @@ -1406,13 +1406,4 @@ public void testExistingRules() throws Exception { update("//pkg:foo"); assertContainsEvent("DEBUG /workspace/pkg/BUILD:5:6: [\"foo\"]"); } - - /** Runs the same test with the Skyframe-based analysis prep. */ - @RunWith(JUnit4.class) - public static class WithSkyframePrepareAnalysis extends BuildViewTest { - @Override - protected FlagBuilder defaultFlags() { - return super.defaultFlags().with(Flag.SKYFRAME_PREPARE_ANALYSIS); - } - } } diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java index a2672c0eafba21..c4ed837cb04297 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java @@ -122,8 +122,6 @@ public abstract class AnalysisTestCase extends FoundationTestCase { public enum Flag { // The --keep_going flag. KEEP_GOING, - // The --skyframe_prepare_analysis flag. - SKYFRAME_PREPARE_ANALYSIS, // Flags for visibility to default to public. PUBLIC_VISIBILITY, // Flags for CPU to work (be set to k8) in test mode. @@ -418,7 +416,6 @@ protected AnalysisResult update( // update --keep_going option if test requested it. boolean keepGoing = flags.contains(Flag.KEEP_GOING); boolean discardAnalysisCache = viewOptions.discardAnalysisCache; - viewOptions.skyframePrepareAnalysis = flags.contains(Flag.SKYFRAME_PREPARE_ANALYSIS); PackageOptions packageOptions = optionsParser.getOptions(PackageOptions.class); PathPackageLocator pathPackageLocator = From 10a9a7aba5a54031c5ebcc3625f05347f60b419b Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 07:35:52 -0700 Subject: [PATCH 32/51] Automated rollback of commit 15c412eb9aa38e1b81f7dd2047849bbb55417a83. *** Reason for rollback *** Roll forward with fix. The original CL caused infinite expansion of rule transitions when they were non-convergent. The fix adds idempotency checking to transitions when they are non-noop and flags them to not apply a rule transition via the new ConfiguredTargetKey.shouldApplyRuleTransition property. This is discussed more thoroughly in the code comments of ConfiguredTargetKey and IdempotencyChecker. *** Original change description *** Automated rollback of commit 52dbdc7a92cedfa212ef681f88e0b733cb5280e0. *** Reason for rollback *** https://github.com/bazelbuild/bazel/issues/18771 PiperOrigin-RevId: 545664983 Change-Id: Id1e46b61506edb861e76ff5f3858e3c95aaaa407 --- .../lib/actions/ActionLookupKeyOrProxy.java | 1 + .../analysis/TransitiveDependencyState.java | 2 +- .../producers/AttributeConfiguration.java | 15 + .../producers/ConfigConditionsProducer.java | 20 +- .../ConfiguredTargetAndDataProducer.java | 26 +- .../producers/DependencyProducer.java | 65 +++- .../producers/PrerequisiteParameters.java | 19 +- .../producers/PrerequisitesProducer.java | 40 ++- .../TargetAndConfigurationProducer.java | 265 ++++++++------ .../skyframe/ConfiguredTargetFunction.java | 8 +- .../lib/skyframe/ConfiguredTargetKey.java | 337 +++++++----------- .../build/lib/skyframe/SkyframeExecutor.java | 7 +- .../google/devtools/build/lib/analysis/BUILD | 1 - .../analysis/ConfigurableAttributesTest.java | 71 ---- .../analysis/util/BuildViewForTesting.java | 2 +- .../lib/skyframe/ConfiguredTargetKeyTest.java | 122 ++----- 16 files changed, 469 insertions(+), 532 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionLookupKeyOrProxy.java b/src/main/java/com/google/devtools/build/lib/actions/ActionLookupKeyOrProxy.java index 92b189cf5e8c3b..48f76839214673 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/ActionLookupKeyOrProxy.java +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionLookupKeyOrProxy.java @@ -29,6 +29,7 @@ * are subclasses of {@link ActionLookupKeyOrProxy}. This allows callers to easily find the value * key, while remaining agnostic to what action lookup values actually exist. */ +// TODO(b/261521010): this layer of indirection is no longer needed and may be cleaned up. public interface ActionLookupKeyOrProxy extends ArtifactOwner { /** * Returns the {@link BuildConfigurationKey} for the configuration associated with this key, or diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TransitiveDependencyState.java b/src/main/java/com/google/devtools/build/lib/analysis/TransitiveDependencyState.java index ac9c872ea089cf..111b352f5ab504 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/TransitiveDependencyState.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/TransitiveDependencyState.java @@ -62,7 +62,7 @@ public final class TransitiveDependencyState { * *

More ideally, those properties would be conveyed via providers of those dependencies, but * doing so would adversely affect resting heap usage whereas {@link ConfiguredTargetAndData} is - * ephemeral. Distributed implementations will include these properties in an extra providers. It + * ephemeral. Distributed implementations will include these properties in an extra provider. It * won't affect memory because the underlying package won't exist on the node loading it remotely. * *

It's valid to obtain {@link Package}s of dependencies from this map instead of creating an diff --git a/src/main/java/com/google/devtools/build/lib/analysis/producers/AttributeConfiguration.java b/src/main/java/com/google/devtools/build/lib/analysis/producers/AttributeConfiguration.java index c927e3c38467a0..88d68224308699 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/producers/AttributeConfiguration.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/producers/AttributeConfiguration.java @@ -14,6 +14,7 @@ package com.google.devtools.build.lib.analysis.producers; import com.google.auto.value.AutoOneOf; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.packages.PackageGroup; import com.google.devtools.build.lib.skyframe.BuildConfigurationKey; @@ -28,6 +29,13 @@ enum Kind { * target is known, it should be verified to be a {@link PackageGroup}. */ VISIBILITY, + /** + * The configuration is null. + * + *

This is only applied when the dependency is in the same package as the parent and it is + * not configurable. The value in this case stores any transition keys. + */ + NULL_TRANSITION_KEYS, /** * There is a single configuration. * @@ -46,6 +54,8 @@ enum Kind { abstract void visibility(); + abstract ImmutableList nullTransitionKeys(); + abstract BuildConfigurationKey unary(); abstract ImmutableMap split(); @@ -53,6 +63,7 @@ enum Kind { public int count() { switch (kind()) { case VISIBILITY: + case NULL_TRANSITION_KEYS: case UNARY: return 1; case SPLIT: @@ -65,6 +76,10 @@ static AttributeConfiguration ofVisibility() { return AutoOneOf_AttributeConfiguration.visibility(); } + static AttributeConfiguration ofNullTransitionKeys(ImmutableList transitionKeys) { + return AutoOneOf_AttributeConfiguration.nullTransitionKeys(transitionKeys); + } + static AttributeConfiguration ofUnary(BuildConfigurationKey key) { return AutoOneOf_AttributeConfiguration.unary(key); } diff --git a/src/main/java/com/google/devtools/build/lib/analysis/producers/ConfigConditionsProducer.java b/src/main/java/com/google/devtools/build/lib/analysis/producers/ConfigConditionsProducer.java index c9d91a559ce06a..b303b9673623fe 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/producers/ConfigConditionsProducer.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/producers/ConfigConditionsProducer.java @@ -13,8 +13,9 @@ // limitations under the License. package com.google.devtools.build.lib.analysis.producers; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.devtools.build.lib.analysis.InvalidVisibilityDependencyException; +import com.google.devtools.build.lib.analysis.InconsistentNullConfigException; import com.google.devtools.build.lib.analysis.TargetAndConfiguration; import com.google.devtools.build.lib.analysis.TransitiveDependencyState; import com.google.devtools.build.lib.analysis.config.ConfigConditions; @@ -99,7 +100,7 @@ public StateMachine step(Tasks tasks, ExtendedEventHandler listener) { .setLabel(configLabels.get(i)) .setConfiguration(targetAndConfiguration.getConfiguration()) .build(), - /* transitionKey= */ null, + /* transitionKeys= */ ImmutableList.of(), transitiveState, (ConfiguredTargetAndDataProducer.ResultSink) this, i)); @@ -130,17 +131,12 @@ public void acceptConfiguredTargetAndDataError(ConfiguredValueCreationException } @Override - public void acceptConfiguredTargetAndDataError(InvalidVisibilityDependencyException error) { - // After removing the rule transition from dependency resolution, a ConfiguredTargetKey in - // Skyframe with a null BuildConfigurationKey will only be used to request visibility - // dependencies. This will never be the case for ConfigConditions, which are always requested - // with the parent configuration. At the moment, nothing throws - // InvalidVisibilityDependencyException. - // - // TODO(b/261521010): update this comment once rule transitions are removed from dependency - // resolution. + public void acceptConfiguredTargetAndDataError(InconsistentNullConfigException error) { + // A config label was evaluated with a null configuration. This should never happen as + // ConfigConditions are only present if the parent is a Rule, then always evaluated with the + // parent configuration. throw new IllegalArgumentException( - "ConfigCondition dependency should never be marked visibility.", error); + "ConfigCondition dependency should never be evaluated with a null configuration.", error); } private StateMachine constructConfigConditions(Tasks tasks, ExtendedEventHandler listener) { diff --git a/src/main/java/com/google/devtools/build/lib/analysis/producers/ConfiguredTargetAndDataProducer.java b/src/main/java/com/google/devtools/build/lib/analysis/producers/ConfiguredTargetAndDataProducer.java index 2ca4a755ff50b2..89f1ed58defd0c 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/producers/ConfiguredTargetAndDataProducer.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/producers/ConfiguredTargetAndDataProducer.java @@ -16,7 +16,7 @@ import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.analysis.ConfiguredTarget; import com.google.devtools.build.lib.analysis.ConfiguredTargetValue; -import com.google.devtools.build.lib.analysis.InvalidVisibilityDependencyException; +import com.google.devtools.build.lib.analysis.InconsistentNullConfigException; import com.google.devtools.build.lib.analysis.TransitiveDependencyState; import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; import com.google.devtools.build.lib.events.ExtendedEventHandler; @@ -42,20 +42,19 @@ public final class ConfiguredTargetAndDataProducer implements StateMachine, Consumer, StateMachine.ValueOrException2Sink< - ConfiguredValueCreationException, InvalidVisibilityDependencyException> { + ConfiguredValueCreationException, InconsistentNullConfigException> { /** Interface for accepting values produced by this class. */ public interface ResultSink { void acceptConfiguredTargetAndData(ConfiguredTargetAndData value, int index); void acceptConfiguredTargetAndDataError(ConfiguredValueCreationException error); - void acceptConfiguredTargetAndDataError(InvalidVisibilityDependencyException error); + void acceptConfiguredTargetAndDataError(InconsistentNullConfigException error); } // -------------------- Input -------------------- private final ConfiguredTargetKey key; - @Nullable // Null if no transition key is needed (patch transition or no-op split transition). - private final String transitionKey; + private final ImmutableList transitionKeys; private final TransitiveDependencyState transitiveState; // -------------------- Output -------------------- @@ -70,12 +69,12 @@ public interface ResultSink { public ConfiguredTargetAndDataProducer( ConfiguredTargetKey key, - @Nullable String transitionKey, + ImmutableList transitionKeys, TransitiveDependencyState transitiveState, ResultSink sink, int outputIndex) { this.key = key; - this.transitionKey = transitionKey; + this.transitionKeys = transitionKeys; this.transitiveState = transitiveState; this.sink = sink; this.outputIndex = outputIndex; @@ -86,9 +85,8 @@ public StateMachine step(Tasks tasks, ExtendedEventHandler listener) { tasks.lookUp( key.toKey(), ConfiguredValueCreationException.class, - InvalidVisibilityDependencyException.class, - (ValueOrException2Sink< - ConfiguredValueCreationException, InvalidVisibilityDependencyException>) + InconsistentNullConfigException.class, + (ValueOrException2Sink) this); return this::fetchConfigurationAndPackage; } @@ -97,7 +95,7 @@ public StateMachine step(Tasks tasks, ExtendedEventHandler listener) { public void acceptValueOrException2( @Nullable SkyValue value, @Nullable ConfiguredValueCreationException error, - @Nullable InvalidVisibilityDependencyException visibilityError) { + @Nullable InconsistentNullConfigException visibilityError) { if (value != null) { var configuredTargetValue = (ConfiguredTargetValue) value; this.configuredTarget = configuredTargetValue.getConfiguredTarget(); @@ -171,11 +169,7 @@ private StateMachine constructResult(Tasks tasks, ExtendedEventHandler listener) throw new IllegalStateException("Target already verified for " + configuredTarget, e); } sink.acceptConfiguredTargetAndData( - new ConfiguredTargetAndData( - configuredTarget, - target, - configurationValue, - transitionKey == null ? ImmutableList.of() : ImmutableList.of(transitionKey)), + new ConfiguredTargetAndData(configuredTarget, target, configurationValue, transitionKeys), outputIndex); return DONE; } diff --git a/src/main/java/com/google/devtools/build/lib/analysis/producers/DependencyProducer.java b/src/main/java/com/google/devtools/build/lib/analysis/producers/DependencyProducer.java index 2940409889ba9d..bc62b56c18507a 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/producers/DependencyProducer.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/producers/DependencyProducer.java @@ -30,11 +30,16 @@ import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; import com.google.devtools.build.lib.analysis.config.DependencyEvaluationException; import com.google.devtools.build.lib.analysis.starlark.StarlarkTransition.TransitionException; +import com.google.devtools.build.lib.causes.LoadingFailedCause; import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.devtools.build.lib.packages.Aspect; import com.google.devtools.build.lib.packages.Attribute; import com.google.devtools.build.lib.packages.AttributeTransitionData; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.TargetUtils; import com.google.devtools.build.lib.skyframe.BuildConfigurationKey; import com.google.devtools.build.lib.skyframe.ConfiguredTargetAndData; import com.google.devtools.build.lib.skyframe.ConfiguredValueCreationException; @@ -183,16 +188,37 @@ private StateMachine processTransitionResult(Tasks tasks, ExtendedEventHandler l return DONE; // There was a previously reported error. } - AttributeConfiguration configuration; - if (transitionedConfigurations.size() == 1 - && transitionedConfigurations.keySet().iterator().next().equals(PATCH_TRANSITION_KEY)) { - // Drops the transition key if it was a patch transition. - configuration = - AttributeConfiguration.ofUnary(transitionedConfigurations.get(PATCH_TRANSITION_KEY)); - } else { - configuration = AttributeConfiguration.ofSplit(transitionedConfigurations); + if (isNonconfigurableTargetInSamePackage(listener)) { + // The target is in the same package as the parent and non-configurable. In the general case + // loading a child target would defeat Package-based sharding. However, when the target is in + // the same Package, that concern no longer applies. This optimization means that delegation, + // and the corresponding creation of additional Skyframe nodes, can be avoided in the very + // common case of source file dependencies in the same Package. + + // Discards transition keys for patch transitions but keeps them otherwise. + ImmutableList transitionKeys = + transitionedConfigurations.size() == 1 + && transitionedConfigurations.containsKey(PATCH_TRANSITION_KEY) + ? ImmutableList.of() + : transitionedConfigurations.keySet().asList(); + return computePrerequisites( + AttributeConfiguration.ofNullTransitionKeys(transitionKeys), + /* executionPlatformLabel= */ null); + } + + if (transitionedConfigurations.size() == 1) { + BuildConfigurationKey patchedConfiguration = + transitionedConfigurations.get(PATCH_TRANSITION_KEY); + if (patchedConfiguration != null) { + // It was a patch transition. Drops the transition key. + return computePrerequisites( + AttributeConfiguration.ofUnary(patchedConfiguration), + /* executionPlatformLabel= */ null); + } } - return computePrerequisites(configuration, /* executionPlatformLabel= */ null); + return computePrerequisites( + AttributeConfiguration.ofSplit(transitionedConfigurations), + /* executionPlatformLabel= */ null); } private StateMachine computePrerequisites( @@ -226,6 +252,27 @@ public void acceptPrerequisitesAspectError(DependencyEvaluationException error) sink.acceptDependencyError(DependencyError.of(error)); } + private boolean isNonconfigurableTargetInSamePackage(ExtendedEventHandler listener) { + Target parentTarget = parameters.target(); + if (parentTarget.getLabel().getPackageIdentifier().equals(toLabel.getPackageIdentifier())) { + try { + Target toTarget = parentTarget.getPackage().getTarget(toLabel.getName()); + if (!toTarget.isConfigurable()) { + return true; + } + } catch (NoSuchTargetException e) { + parameters + .transitiveState() + .addTransitiveCause(new LoadingFailedCause(toLabel, e.getDetailedExitCode())); + listener.handle( + Event.error( + TargetUtils.getLocationMaybe(parentTarget), + TargetUtils.formatMissingEdge(parentTarget, toLabel, e, kind.getAttribute()))); + } + } + return false; + } + /** * Emits errors from {@link ExecutionPlatformResult#error}. * diff --git a/src/main/java/com/google/devtools/build/lib/analysis/producers/PrerequisiteParameters.java b/src/main/java/com/google/devtools/build/lib/analysis/producers/PrerequisiteParameters.java index 4cdd28db3de305..7ab5fd1300888c 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/producers/PrerequisiteParameters.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/producers/PrerequisiteParameters.java @@ -25,6 +25,7 @@ import com.google.devtools.build.lib.packages.Aspect; import com.google.devtools.build.lib.packages.ConfiguredAttributeMapper; import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; import com.google.devtools.build.lib.skyframe.BuildConfigurationKey; import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey; import javax.annotation.Nullable; @@ -33,7 +34,7 @@ /** Common parameters for computing prerequisites. */ public final class PrerequisiteParameters { private final ConfiguredTargetKey configuredTargetKey; - @Nullable private final Rule associatedRule; + private final Target target; private final ImmutableList aspects; private final StarlarkTransitionCache transitionCache; @@ -44,14 +45,14 @@ public final class PrerequisiteParameters { public PrerequisiteParameters( ConfiguredTargetKey configuredTargetKey, - @Nullable Rule associatedRule, + Target target, Iterable aspects, StarlarkTransitionCache transitionCache, @Nullable ToolchainCollection toolchainContexts, @Nullable ConfiguredAttributeMapper attributeMap, TransitiveDependencyState transitiveState) { this.configuredTargetKey = configuredTargetKey; - this.associatedRule = associatedRule; + this.target = target; this.aspects = ImmutableList.copyOf(aspects); this.transitionCache = transitionCache; this.toolchainContexts = toolchainContexts; @@ -63,9 +64,13 @@ public Label label() { return configuredTargetKey.getLabel(); } + public Target target() { + return target; + } + @Nullable public Rule associatedRule() { - return associatedRule; + return target.getAssociatedRule(); } @Nullable @@ -91,12 +96,8 @@ public ConfiguredAttributeMapper attributeMap() { return attributeMap; } - @Nullable public Location location() { - if (associatedRule == null) { - return null; - } - return associatedRule.getLocation(); + return target.getLocation(); } public BuildEventId eventId() { diff --git a/src/main/java/com/google/devtools/build/lib/analysis/producers/PrerequisitesProducer.java b/src/main/java/com/google/devtools/build/lib/analysis/producers/PrerequisitesProducer.java index 295eedb29706f3..5f9fbe96689510 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/producers/PrerequisitesProducer.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/producers/PrerequisitesProducer.java @@ -15,6 +15,7 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.devtools.build.lib.analysis.AspectResolutionHelpers.computeAspectCollection; +import static com.google.devtools.build.lib.analysis.producers.AttributeConfiguration.Kind.VISIBILITY; import static com.google.devtools.build.lib.skyframe.ConfiguredTargetAndData.SPLIT_DEP_ORDERING; import static java.util.Arrays.sort; @@ -22,9 +23,11 @@ import com.google.devtools.build.lib.analysis.AspectCollection; import com.google.devtools.build.lib.analysis.DuplicateException; import com.google.devtools.build.lib.analysis.InconsistentAspectOrderException; +import com.google.devtools.build.lib.analysis.InconsistentNullConfigException; import com.google.devtools.build.lib.analysis.InvalidVisibilityDependencyException; import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; import com.google.devtools.build.lib.analysis.config.DependencyEvaluationException; +import com.google.devtools.build.lib.analysis.configuredtargets.PackageGroupConfiguredTarget; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.devtools.build.lib.packages.Aspect; @@ -104,7 +107,16 @@ public StateMachine step(Tasks tasks, ExtendedEventHandler listener) { tasks.enqueue( new ConfiguredTargetAndDataProducer( getPrerequisiteKey(/* configurationKey= */ null), - /* transitionKey= */ null, + /* transitionKeys= */ ImmutableList.of(), + parameters.transitiveState(), + (ConfiguredTargetAndDataProducer.ResultSink) this, + /* outputIndex= */ 0)); + break; + case NULL_TRANSITION_KEYS: + tasks.enqueue( + new ConfiguredTargetAndDataProducer( + getPrerequisiteKey(/* configurationKey= */ null), + configuration.nullTransitionKeys(), parameters.transitiveState(), (ConfiguredTargetAndDataProducer.ResultSink) this, /* outputIndex= */ 0)); @@ -113,7 +125,7 @@ public StateMachine step(Tasks tasks, ExtendedEventHandler listener) { tasks.enqueue( new ConfiguredTargetAndDataProducer( getPrerequisiteKey(configuration.unary()), - /* transitionKey= */ null, + /* transitionKeys= */ ImmutableList.of(), parameters.transitiveState(), (ConfiguredTargetAndDataProducer.ResultSink) this, /* outputIndex= */ 0)); @@ -124,7 +136,7 @@ public StateMachine step(Tasks tasks, ExtendedEventHandler listener) { tasks.enqueue( new ConfiguredTargetAndDataProducer( getPrerequisiteKey(entry.getValue()), - /* transitionKey= */ entry.getKey(), + ImmutableList.of(entry.getKey()), parameters.transitiveState(), (ConfiguredTargetAndDataProducer.ResultSink) this, index)); @@ -141,9 +153,18 @@ public void acceptConfiguredTargetAndData(ConfiguredTargetAndData value, int ind } @Override - public void acceptConfiguredTargetAndDataError(InvalidVisibilityDependencyException error) { + public void acceptConfiguredTargetAndDataError(InconsistentNullConfigException error) { hasError = true; - sink.acceptPrerequisitesError(error); + if (configuration.kind() == VISIBILITY) { + // The target was configurable, but used as a visibility dependency. This is invalid because + // only `PackageGroup`s are accepted as visibility dependencies and those are not + // configurable. Propagates the exception with more precise information. + sink.acceptPrerequisitesError(new InvalidVisibilityDependencyException(label)); + return; + } + // `configuration.kind()` was `NULL_TRANSITION_KEYS`. This is only used when the target is in + // the same package as the parent and not configurable so this should never happen. + throw new IllegalStateException(error); } @Override @@ -157,6 +178,15 @@ private StateMachine computeConfiguredAspects(Tasks tasks, ExtendedEventHandler return DONE; } + if (configuration.kind() == VISIBILITY) { + // Verifies that the dependency is a `package_group`. The value is always at index 0 because + // the `VISIBILITY` configuration is always unary. + if (!(configuredTargets[0].getConfiguredTarget() instanceof PackageGroupConfiguredTarget)) { + sink.acceptPrerequisitesError(new InvalidVisibilityDependencyException(label)); + return DONE; + } + } + cleanupValues(); AspectCollection aspects; diff --git a/src/main/java/com/google/devtools/build/lib/analysis/producers/TargetAndConfigurationProducer.java b/src/main/java/com/google/devtools/build/lib/analysis/producers/TargetAndConfigurationProducer.java index c8fdc99f0de7c1..de3b1d8e52f28e 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/producers/TargetAndConfigurationProducer.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/producers/TargetAndConfigurationProducer.java @@ -22,7 +22,6 @@ import com.google.devtools.build.lib.actions.ActionLookupKey; import com.google.devtools.build.lib.analysis.ConfiguredTargetValue; import com.google.devtools.build.lib.analysis.InconsistentNullConfigException; -import com.google.devtools.build.lib.analysis.InvalidVisibilityDependencyException; import com.google.devtools.build.lib.analysis.TargetAndConfiguration; import com.google.devtools.build.lib.analysis.TransitiveDependencyState; import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; @@ -35,10 +34,10 @@ import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.devtools.build.lib.packages.NoSuchPackageException; import com.google.devtools.build.lib.packages.NoSuchTargetException; -import com.google.devtools.build.lib.packages.PackageGroup; import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.packages.RuleTransitionData; import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.TargetUtils; import com.google.devtools.build.lib.skyframe.BuildConfigurationKey; import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey; import com.google.devtools.build.lib.skyframe.ConfiguredValueCreationException; @@ -54,16 +53,14 @@ * Computes the target and configuration for a configured target key. * *

If the key has a configuration and the target is configurable, attempts to apply a rule side - * transition. If the target is not configurable, directly transitions to the null configuration. If - * the resulting configuration already has an owner, delegates to the owner instead of recomputing - * the configured target. - * - *

If the key does not have a configuration, it was requested as a visibility dependency. - * Verifies that the {@link Target} is a {@link PackageGroup}, throwing {@link - * InvalidVisibilityDependencyException} if that's not the case. + * transition. If the configuration changes, delegates to a target with the new configuration. If + * the target is not configurable, directly delegates to the null configuration. */ public final class TargetAndConfigurationProducer - implements StateMachine, Consumer, TargetProducer.ResultSink { + implements StateMachine, + StateMachine.ValueOrExceptionSink, + Consumer, + TargetProducer.ResultSink { /** Accepts results of this producer. */ public interface ResultSink { void acceptTargetAndConfiguration(TargetAndConfiguration value, ConfiguredTargetKey fullKey); @@ -79,7 +76,6 @@ public abstract static class TargetAndConfigurationError { /** Tags the error type. */ public enum Kind { CONFIGURED_VALUE_CREATION, - INVALID_VISIBILITY_DEPENDENCY, INCONSISTENT_NULL_CONFIG } @@ -87,8 +83,6 @@ public enum Kind { public abstract ConfiguredValueCreationException configuredValueCreation(); - public abstract InvalidVisibilityDependencyException invalidVisibilityDependency(); - public abstract InconsistentNullConfigException inconsistentNullConfig(); private static TargetAndConfigurationError of(ConfiguredValueCreationException e) { @@ -96,15 +90,6 @@ private static TargetAndConfigurationError of(ConfiguredValueCreationException e .configuredValueCreation(e); } - // TODO(b/261521010): enable this error once Rule transitions are removed from dependency - // resolution. - // private static TargetAndConfigurationError of(InvalidVisibilityDependencyException e) { - // return AutoOneOf_TargetAndConfigurationProducer_TargetAndConfigurationError - // .invalidVisibilityDependency(e); - // } - - // TODO(b/261521010): delete this error once Rule transitions are removed from dependency - // resolution. private static TargetAndConfigurationError of(InconsistentNullConfigException e) { return AutoOneOf_TargetAndConfigurationProducer_TargetAndConfigurationError .inconsistentNullConfig(e); @@ -173,65 +158,96 @@ private StateMachine determineConfiguration(Tasks tasks, ExtendedEventHandler li if (configurationKey == null) { if (target.isConfigurable()) { // We somehow ended up in a target that requires a non-null configuration but with a key - // that doesn't have a configuration. This is always an error, but we need to analyze the - // dependencies of the latter target to realize that. Short-circuit the evaluation to avoid - // doing useless work and running code with a null configuration that's not prepared for it. + // that doesn't have a configuration. This is always an error, but we need to bubble this + // up to the parent to provide more context. sink.acceptTargetAndConfigurationError( TargetAndConfigurationError.of(new InconsistentNullConfigException())); return DONE; } - // TODO(b/261521010): after removing the rule transition from dependency resolution, the logic - // here changes. - // - // A null configuration key will only be used for visibility dependencies so when that's - // true, a check that the target is a PackageGroup will be performed, throwing - // InvalidVisibilityDependencyException on failure. - // - // The ConfiguredTargetKey cannot fan-in in this case. sink.acceptTargetAndConfiguration( new TargetAndConfiguration(target, /* configuration= */ null), preRuleTransitionKey); return DONE; } - // This may happen for top-level ConfiguredTargets. - // - // TODO(b/261521010): this may also happen for targets that are not top-level after removing - // rule transitions from dependency resolution. Update this comment. if (!target.isConfigurable()) { - var nullConfiguredTargetKey = - ConfiguredTargetKey.builder().setDelegate(preRuleTransitionKey).build(); - ActionLookupKey delegate = nullConfiguredTargetKey.toKey(); - if (!delegate.equals(preRuleTransitionKey)) { - // Delegates to the key that already owns the null configuration. - delegateTo(tasks, delegate); - return DONE; - } - sink.acceptTargetAndConfiguration( - new TargetAndConfiguration(target, /* configuration= */ null), nullConfiguredTargetKey); + // If target is not configurable, but requested with a configuration. Delegates to a key with + // the null configuration. This is expected to be uncommon. The common case of a + // non-configurable target is an input file, but those are usually package local and requested + // correctly with the null configuration. + delegateTo( + tasks, + ConfiguredTargetKey.builder() + .setLabel(preRuleTransitionKey.getLabel()) + .setExecutionPlatformLabel(preRuleTransitionKey.getExecutionPlatformLabel()) + .build() + .toKey()); return DONE; } - return new RuleTransitionApplier(); + if (!preRuleTransitionKey.shouldApplyRuleTransition()) { + lookUpConfigurationValue(tasks); + return DONE; + } + + ConfigurationTransition transition = + computeTransition(target.getAssociatedRule(), trimmingTransitionFactory); + if (transition == null) { + lookUpConfigurationValue(tasks); + return DONE; + } + + return new RuleTransitionApplier(transition); + } + + private void delegateTo(Tasks tasks, ActionLookupKey delegate) { + tasks.lookUp(delegate, (Consumer) this); + } + + @Override + public void accept(SkyValue value) { + sink.acceptTargetAndConfigurationDelegatedValue((ConfiguredTargetValue) value); + } + + private void lookUpConfigurationValue(Tasks tasks) { + tasks.lookUp( + preRuleTransitionKey.getConfigurationKey(), + InvalidConfigurationException.class, + (ValueOrExceptionSink) this); } - /** Applies any requested rule transition before producing the final configuration. */ - private class RuleTransitionApplier - implements StateMachine, - TransitionApplier.ResultSink, - ValueOrExceptionSink { + @Override + public void acceptValueOrException( + @Nullable SkyValue value, @Nullable InvalidConfigurationException error) { + if (value != null) { + sink.acceptTargetAndConfiguration( + new TargetAndConfiguration(target, (BuildConfigurationValue) value), + preRuleTransitionKey); + return; + } + emitError( + error.getMessage(), TargetUtils.getLocationMaybe(target), error.getDetailedExitCode()); + } + + /** + * Applies the requested rule transition. + * + *

When the rule transition results in a new configuration, performs an idempotency check and + * constructs a delegate {@link ConfiguredTargetKey} with the appropriate {@link + * ConfiguredTargetKey#shouldApplyRuleTransition} value. Otherwise, just looks up the + * configuration. + */ + private class RuleTransitionApplier implements StateMachine, TransitionApplier.ResultSink { + // -------------------- Input -------------------- + private final ConfigurationTransition transition; // -------------------- Internal State -------------------- private BuildConfigurationKey configurationKey; - private ConfiguredTargetKey fullKey; + + private RuleTransitionApplier(ConfigurationTransition transition) { + this.transition = transition; + } @Override public StateMachine step(Tasks tasks, ExtendedEventHandler listener) { - ConfigurationTransition transition = - computeTransition(target.getAssociatedRule(), trimmingTransitionFactory); - if (transition == null) { - this.configurationKey = preRuleTransitionKey.getConfigurationKey(); - return processTransitionedKey(tasks, listener); - } - return new TransitionApplier( preRuleTransitionKey.getConfigurationKey(), transition, @@ -266,57 +282,104 @@ private StateMachine processTransitionedKey(Tasks tasks, ExtendedEventHandler li return DONE; // There was an error. } - if (!configurationKey.equals(preRuleTransitionKey.getConfigurationKey())) { - fullKey = - ConfiguredTargetKey.builder() - .setDelegate(preRuleTransitionKey) - .setConfigurationKey(configurationKey) - .build(); - ActionLookupKey delegate = fullKey.toKey(); - if (!delegate.equals(preRuleTransitionKey)) { - // Delegates to the key that already owns this configuration. - delegateTo(tasks, delegate); - return DONE; - } - } else { - fullKey = preRuleTransitionKey; + if (configurationKey.equals(preRuleTransitionKey.getConfigurationKey())) { + // This key owns the configuration and the computation completes normally. + lookUpConfigurationValue(tasks); + return DONE; } - // This key owns the configuration and the computation completes normally. - tasks.lookUp( - configurationKey, - InvalidConfigurationException.class, - (ValueOrExceptionSink) this); - return DONE; + return new IdempotencyChecker(); } - @Override - public void acceptValueOrException( - @Nullable SkyValue value, @Nullable InvalidConfigurationException error) { - if (value != null) { - sink.acceptTargetAndConfiguration( - new TargetAndConfiguration(target, (BuildConfigurationValue) value), fullKey); - return; + /** + * Checks the transition for idempotency before applying delegation. + * + *

If the transition is non-idempotent, marks {@link + * ConfiguredTargetKey#shouldApplyRuleTransition} false in the delegate key. + */ + private class IdempotencyChecker implements StateMachine, TransitionApplier.ResultSink { + /* At first glance, it seems like setting `shouldApplyRuleTransition=false` should be benign + * in both cases, but it would be an error in the idempotent case. + * + * Idempotent Case + * + * If we were to mark the idempotent case with `shouldApplyRuleTransition=false`, it would + * lead to action conflicts. Let `//foo[123]` be a key that rule transitions to `//foo[abc]` + * and suppose the outcome is marked `//foo[abc] shouldApplyRuleTransition=false`. + * + * A different parent might directly request `//foo[abc] shouldApplyRuleTransition=true`. + * Since the rule transition is a idempotent, it would result in the same actions as + * `//foo[abc] shouldApplyRuleTransition=false` with a different key, causing action + * conflicts. + * + * Non-idempotent Case + * + * In the example of //foo[abc] shouldApplyRuleTransition=false and //foo[abc] + * shouldApplyRuleTransition=true, there should be no action conflicts because the + * `shouldApplyRuleTransition=false` is the result of a non-idempotent rule transition and + * `shouldApplyRuleTransition=true` will produce a different configuration. */ + + // -------------------- Internal State -------------------- + private BuildConfigurationKey configurationKey2; + + @Override + public StateMachine step(Tasks tasks, ExtendedEventHandler listener) { + return new TransitionApplier( + configurationKey, + transition, + transitionCache, + (TransitionApplier.ResultSink) this, + /* runAfter= */ this::checkIdempotencyAndDelegate); + } + + @Override + public void acceptTransitionedConfigurations( + ImmutableMap transitionResult) { + checkState( + transitionResult.size() == 1, "Expected exactly one result: %s", transitionResult); + this.configurationKey2 = + checkNotNull( + transitionResult.get(ConfigurationTransition.PATCH_TRANSITION_KEY), + "Transition result missing patch transition entry: %s", + transitionResult); + } + + @Override + public void acceptTransitionError(TransitionException e) { + emitTransitionErrorMessage(e.getMessage()); + } + + @Override + public void acceptTransitionError(OptionsParsingException e) { + emitTransitionErrorMessage(e.getMessage()); + } + + private StateMachine checkIdempotencyAndDelegate(Tasks tasks, ExtendedEventHandler listener) { + if (configurationKey2 == null) { + return DONE; // There was an error. + } + + ConfiguredTargetKey.Builder keyBuilder = + ConfiguredTargetKey.builder() + .setLabel(preRuleTransitionKey.getLabel()) + .setExecutionPlatformLabel(preRuleTransitionKey.getExecutionPlatformLabel()) + .setConfigurationKey(configurationKey); + + if (!configurationKey.equals(configurationKey2)) { + // The transition was not idempotent. Explicitly informs the delegate to avoid applying a + // rule transition. + keyBuilder.setShouldApplyRuleTransition(false); + } + delegateTo(tasks, keyBuilder.build().toKey()); + return DONE; } - emitTransitionErrorMessage(error.getMessage()); } private void emitTransitionErrorMessage(String message) { - // The target must be a rule because these errors happen during the Rule transition. - Rule rule = target.getAssociatedRule(); - emitError(message, rule.getLocation(), /* exitCode= */ null); + emitError(message, TargetUtils.getLocationMaybe(target), /* exitCode= */ null); } } - private void delegateTo(Tasks tasks, ActionLookupKey delegate) { - tasks.lookUp(delegate, (Consumer) this); - } - - @Override - public void accept(SkyValue value) { - sink.acceptTargetAndConfigurationDelegatedValue((ConfiguredTargetValue) value); - } - private void emitError( String message, @Nullable Location location, @Nullable DetailedExitCode exitCode) { sink.acceptTargetAndConfigurationError( diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java index 8fa7842a22039d..9b9756ce198f20 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java @@ -228,7 +228,6 @@ public SkyValue compute(SkyKey key, Environment env) throws ReportedException, UnreportedException, DependencyException, InterruptedException { State state = env.getState(() -> new State(storeTransitivePackages)); ConfiguredTargetKey configuredTargetKey = (ConfiguredTargetKey) key.argument(); - Preconditions.checkArgument(!configuredTargetKey.isProxy(), configuredTargetKey); SkyframeBuildView view = buildViewProvider.getSkyframeBuildView(); if (shouldUnblockCpuWorkWhenFetchingDeps) { @@ -249,6 +248,9 @@ public SkyValue compute(SkyKey key, Environment env) } catch (InconsistentNullConfigException e) { // TODO(b/267529852): see if we can remove this. It's not clear the conditions that trigger // InconsistentNullConfigException are even possible. + // + // TODO(b/261521010): propagate this exception once the parent side rule transition is + // deleted. The caller should handle it correctly. return new NonRuleConfiguredTargetValue( new EmptyConfiguredTarget(configuredTargetKey), computeDependenciesState.transitivePackages()); @@ -507,10 +509,6 @@ private void computeTargetAndConfiguration( storedEvents.handle(Event.error(e.getLocation(), e.getMessage())); } throw new ReportedException(e); - case INVALID_VISIBILITY_DEPENDENCY: - // Bubbles the error up to the parent ConfiguredTargetFunction where it will be reported - // with additional context. - throw new DependencyException(error.invalidVisibilityDependency()); case INCONSISTENT_NULL_CONFIG: throw error.inconsistentNullConfig(); } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKey.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKey.java index e2282048cbbc6e..e9e28e3a7ad68e 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKey.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKey.java @@ -14,13 +14,11 @@ package com.google.devtools.build.lib.skyframe; -import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.devtools.build.lib.util.HashCodes.hashObjects; import com.google.common.base.MoreObjects; import com.google.devtools.build.lib.actions.ActionLookupKey; -import com.google.devtools.build.lib.actions.ActionLookupKeyOrProxy; import com.google.devtools.build.lib.analysis.ConfiguredTarget; import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; import com.google.devtools.build.lib.cmdline.Label; @@ -43,17 +41,9 @@ * dependency resolution and the rule analysis. * *

In practice, a ({@link Label} and post-transition {@link BuildConfigurationKey}) pair plus a - * possible execution platform override {@link Label} with special constraints. To elaborate, in - * order of highest to lowest potential for concern: + * possible execution platform override {@link Label} with special constraints described as follows. * - *

1. The {@link BuildConfigurationKey} must be post-transition and thus ready for immediate use - * in dependency resolution and analysis. In practice, this means that if the rule has an - * incoming-edge transition (cfg in {@link RuleClass}) or there are global trimming transitions, - * THOSE TRANSITIONS MUST ALREADY BE DONE before creating the key. Failure to do so will lead to - * build graphs with ConfiguredTarget that have seemingly impossible {@link BuildConfigurationValue} - * (due to the skipped transitions). - * - *

2. A build should not request keys with equal ({@link Label}, {@link BuildConfigurationValue}) + *

A build should not request keys with equal ({@link Label}, {@link BuildConfigurationValue}) * pairs but different execution platform override {@link Label} if the invoked rule will register * actions. (This is potentially OK if all outputs of all registered actions incorporate the * execution platform in their name unless the build also requests keys without an override that @@ -63,32 +53,40 @@ * *

Note that this key may be used to look up the generating action of an artifact. * - *

The {@link ConfiguredTargetKey} is not a {@link SkyKey} and must be cast to one using {@link - * ActionLookupKeyOrProxy#toKey}. - * *

TODO(blaze-configurability-team): Consider just using BuildOptions over a * BuildConfigurationKey. */ -public abstract class ConfiguredTargetKey implements ActionLookupKeyOrProxy { +public class ConfiguredTargetKey implements ActionLookupKey { /** * Cache so that the number of ConfiguredTargetKey instances is {@code O(configured targets)} and * not {@code O(edges between configured targets)}. */ - private static final SkyKey.SkyKeyInterner interner = SkyKey.newInterner(); + private static final SkyKey.SkyKeyInterner interner = SkyKey.newInterner(); + private final Label label; @Nullable private final BuildConfigurationKey configurationKey; private final transient int hashCode; - private ConfiguredTargetKey(@Nullable BuildConfigurationKey configurationKey, int hashCode) { + private ConfiguredTargetKey( + Label label, @Nullable BuildConfigurationKey configurationKey, int hashCode) { + this.label = label; this.configurationKey = configurationKey; this.hashCode = hashCode; } - public Builder toBuilder() { - return builder() - .setConfigurationKey(configurationKey) - .setLabel(getLabel()) - .setExecutionPlatformLabel(getExecutionPlatformLabel()); + @Override + public final SkyFunctionName functionName() { + return SkyFunctions.CONFIGURED_TARGET; + } + + @Override + public SkyKeyInterner getSkyKeyInterner() { + return interner; + } + + @Override + public Label getLabel() { + return label; } @Nullable @@ -97,22 +95,36 @@ public final BuildConfigurationKey getConfigurationKey() { return configurationKey; } - public abstract Label getExecutionPlatformLabel(); - @Override - public final int hashCode() { - return hashCode; + public final ConfiguredTargetKey toKey() { + return this; } - public boolean isProxy() { - return false; + @Nullable + public Label getExecutionPlatformLabel() { + return null; } - private static int computeHashCode( - Label label, - @Nullable BuildConfigurationKey configurationKey, - @Nullable Label executionPlatformLabel) { - return hashObjects(label, configurationKey, executionPlatformLabel); + /** + * True if the target's rule transition should be applied. + * + *

True by default but set false when a non-idempotent rule transition is detected. It prevents + * over-application of such transitions. + */ + public boolean shouldApplyRuleTransition() { + return true; + } + + public final String prettyPrint() { + if (getLabel() == null) { + return "null"; + } + return String.format("%s (%s)", getLabel(), formatConfigurationKey(configurationKey)); + } + + @Override + public final int hashCode() { + return hashCode; } @Override @@ -127,22 +139,12 @@ public final boolean equals(Object obj) { return hashCode == other.hashCode && getLabel().equals(other.getLabel()) && Objects.equals(configurationKey, other.configurationKey) - && Objects.equals(getExecutionPlatformLabel(), other.getExecutionPlatformLabel()); - } - - public String prettyPrint() { - if (getLabel() == null) { - return "null"; - } - return String.format("%s (%s)", getLabel(), formatConfigurationKey(configurationKey)); - } - - private static ConfiguredTargetKey intern(ConfiguredTargetKey key) { - return (ConfiguredTargetKey) interner.intern((SkyKey) key); + && Objects.equals(getExecutionPlatformLabel(), other.getExecutionPlatformLabel()) + && shouldApplyRuleTransition() == other.shouldApplyRuleTransition(); } @Override - public String toString() { + public final String toString() { // TODO(b/162809183): consider reverting to less verbose toString when bug is resolved. MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper(this).add("label", getLabel()).add("config", configurationKey); @@ -152,47 +154,43 @@ public String toString() { return helper.toString(); } - private static final class RealConfiguredTargetKey extends ConfiguredTargetKey - implements ActionLookupKey { - private final Label label; - - private RealConfiguredTargetKey( - Label label, @Nullable BuildConfigurationKey configurationKey, int hashCode) { - super(configurationKey, hashCode); - this.label = label; - } - - static ConfiguredTargetKey create( - Label label, @Nullable BuildConfigurationKey configurationKey) { - int hashCode = computeHashCode(label, configurationKey, /* executionPlatformLabel= */ null); - return intern(new RealConfiguredTargetKey(label, configurationKey, hashCode)); - } - - @Override - public final SkyFunctionName functionName() { - return SkyFunctions.CONFIGURED_TARGET; - } - - @Override - public SkyKeyInterner getSkyKeyInterner() { - return interner; - } + /** + * Key indicating that no rule transition should be applied to the configuration. + * + *

NOTE: although it's true that no rule transition is applied when there is a null + * configuration, this key type is used to handle a special edge case described below. It should + * only be used with a non-null configuration. + * + *

When a non-noop rule transition occurs, it creates a new delegation {@link + * ConfiguredTargetKey} with the resulting configuration. This is so if different starting + * configurations result in the same configuration after transition, they converge on the same + * key-value entry in Skyframe. + * + *

This can be problematic when transitions are not idempotent because evaluation of the + * delegate repeats the transition, resulting in a another delegate. In cases of + * non-convergent transitions, this may lead to infinite expansion. + * + *

To ensure that transitions are effectively only applied once, prior to delegation, the + * {@link ConfiguredTargetFunction} applies the transition a second time to check it for + * idempotency. It sets {@link ConfiguredTargetKey#shouldApplyRuleTransition} false when it is not + * idempotent. + */ + private static final class ConfiguredTargetKeyWithFinalConfiguration extends ConfiguredTargetKey { + // This is implemented using subtypes instead of adding a boolean field to `ConfiguredTargetKey` + // to reduce memory cost. - @Override - public Label getLabel() { - return label; + private ConfiguredTargetKeyWithFinalConfiguration( + Label label, BuildConfigurationKey configurationKey, int hashCode) { + super(label, checkNotNull(configurationKey), hashCode); } - @Nullable @Override - public Label getExecutionPlatformLabel() { - return null; + public boolean shouldApplyRuleTransition() { + return false; } } - private static final class ToolchainDependencyConfiguredTargetKey extends ConfiguredTargetKey - implements ActionLookupKey { - private final Label label; + private static class ToolchainDependencyConfiguredTargetKey extends ConfiguredTargetKey { private final Label executionPlatformLabel; private ToolchainDependencyConfiguredTargetKey( @@ -200,119 +198,38 @@ private ToolchainDependencyConfiguredTargetKey( @Nullable BuildConfigurationKey configurationKey, int hashCode, Label executionPlatformLabel) { - super(configurationKey, hashCode); - this.label = label; + super(label, configurationKey, hashCode); this.executionPlatformLabel = checkNotNull(executionPlatformLabel); } - private static ConfiguredTargetKey create( - Label label, - @Nullable BuildConfigurationKey configurationKey, - Label executionPlatformLabel) { - int hashCode = computeHashCode(label, configurationKey, executionPlatformLabel); - return intern( - new ToolchainDependencyConfiguredTargetKey( - label, configurationKey, hashCode, executionPlatformLabel)); - } - - @Override - public SkyFunctionName functionName() { - return SkyFunctions.CONFIGURED_TARGET; - } - - @Override - public Label getLabel() { - return label; - } - @Override public Label getExecutionPlatformLabel() { return executionPlatformLabel; } - - @Override - public SkyKeyInterner getSkyKeyInterner() { - return interner; - } } - // This class implements SkyKey only so that it can share the interner. It should never be used as - // a SkyKey. - private static final class ProxyConfiguredTargetKey extends ConfiguredTargetKey - implements SkyKey { - private final ConfiguredTargetKey delegate; - - private static ConfiguredTargetKey create( - ConfiguredTargetKey delegate, @Nullable BuildConfigurationKey configurationKey) { - int hashCode = - computeHashCode( - delegate.getLabel(), configurationKey, delegate.getExecutionPlatformLabel()); - return intern(new ProxyConfiguredTargetKey(delegate, configurationKey, hashCode)); - } - - private ProxyConfiguredTargetKey( - ConfiguredTargetKey delegate, - @Nullable BuildConfigurationKey configurationKey, - int hashCode) { - super(configurationKey, hashCode); - checkArgument( - !delegate.isProxy(), "Proxy keys must not be nested: %s %s", delegate, configurationKey); - this.delegate = delegate; - } - - @Override - public SkyFunctionName functionName() { - // ProxyConfiguredTargetKey is never used directly by Skyframe. It must always be cast using - // toKey. - throw new UnsupportedOperationException(); - } - - @Override - public Label getLabel() { - return delegate.getLabel(); - } - - @Override - @Nullable - public Label getExecutionPlatformLabel() { - return delegate.getExecutionPlatformLabel(); - } - - @Override - public ActionLookupKey toKey() { - return (ActionLookupKey) delegate; - } - - @Override - public boolean isProxy() { - return true; - } - - @Override - public Builder toBuilder() { - return new Builder().setDelegate(delegate).setConfigurationKey(getConfigurationKey()); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("delegate", delegate) - .add("config", getConfigurationKey()) - .toString(); + private static final class ToolchainDependencyConfiguredTargetKeyWithFinalConfiguration + extends ToolchainDependencyConfiguredTargetKey { + private ToolchainDependencyConfiguredTargetKeyWithFinalConfiguration( + Label label, + BuildConfigurationKey configurationKey, + int hashCode, + Label executionPlatformLabel) { + super(label, checkNotNull(configurationKey), hashCode, executionPlatformLabel); } @Override - public String prettyPrint() { - return super.prettyPrint() - + " virtual(" - + formatConfigurationKey(getConfigurationKey()) - + ")"; + public boolean shouldApplyRuleTransition() { + return false; } + } - @Override - public SkyKeyInterner getSkyKeyInterner() { - return interner; - } + public Builder toBuilder() { + return builder() + .setConfigurationKey(configurationKey) + .setLabel(getLabel()) + .setExecutionPlatformLabel(getExecutionPlatformLabel()) + .setShouldApplyRuleTransition(shouldApplyRuleTransition()); } /** Returns a new {@link Builder} to create instances of {@link ConfiguredTargetKey}. */ @@ -335,7 +252,7 @@ public static final class Builder { private Label label = null; private BuildConfigurationKey configurationKey = null; private Label executionPlatformLabel = null; - private ConfiguredTargetKey delegate; + private boolean shouldApplyRuleTransition = true; private Builder() {} @@ -369,31 +286,45 @@ public Builder setExecutionPlatformLabel(@Nullable Label executionPlatformLabel) return this; } - /** - * If set, creates a {@link ProxyConfiguredTargetKey}. - * - *

It's invalid to set a label or execution platform label if this is set. Those will be - * defined by the corresponding values of {@code delegate}. - */ @CanIgnoreReturnValue - public Builder setDelegate(ConfiguredTargetKey delegate) { - this.delegate = delegate; + public Builder setShouldApplyRuleTransition(boolean shouldApplyRuleTransition) { + this.shouldApplyRuleTransition = shouldApplyRuleTransition; return this; } /** Builds a new {@link ConfiguredTargetKey} based on the supplied data. */ public ConfiguredTargetKey build() { - if (this.delegate != null) { - checkArgument(label == null); - checkArgument(executionPlatformLabel == null); - return ProxyConfiguredTargetKey.create(delegate, configurationKey); - } - if (this.executionPlatformLabel != null) { - return ToolchainDependencyConfiguredTargetKey.create( - label, configurationKey, executionPlatformLabel); + int hashCode = + computeHashCode( + label, configurationKey, executionPlatformLabel, shouldApplyRuleTransition); + ConfiguredTargetKey newKey; + if (executionPlatformLabel == null) { + newKey = + shouldApplyRuleTransition + ? new ConfiguredTargetKey(label, configurationKey, hashCode) + : new ConfiguredTargetKeyWithFinalConfiguration(label, configurationKey, hashCode); + } else { + newKey = + shouldApplyRuleTransition + ? new ToolchainDependencyConfiguredTargetKey( + label, configurationKey, hashCode, executionPlatformLabel) + : new ToolchainDependencyConfiguredTargetKeyWithFinalConfiguration( + label, configurationKey, hashCode, executionPlatformLabel); } - return RealConfiguredTargetKey.create(label, configurationKey); + return interner.intern(newKey); + } + } + + private static int computeHashCode( + Label label, + @Nullable BuildConfigurationKey configurationKey, + @Nullable Label executionPlatformLabel, + boolean shouldApplyRuleTransition) { + int hashCode = hashObjects(label, configurationKey, executionPlatformLabel); + if (!shouldApplyRuleTransition) { + hashCode = ~hashCode; } + return hashCode; } private static String formatConfigurationKey(@Nullable BuildConfigurationKey key) { @@ -403,13 +334,7 @@ private static String formatConfigurationKey(@Nullable BuildConfigurationKey key return key.getOptions().checksum(); } - /** - * Codec for all {@link ConfiguredTargetKey} subtypes. - * - *

By design, {@link ProxyConfiguredTargetKey} serializes as a key without delegation. Upon - * deserialization, if the key is locally delegated, it becomes delegating again due to interning. - * If not, it deserializes to the appropriate non-delegating key. - */ + /** Codec for all {@link ConfiguredTargetKey} subtypes. */ @Keep private static class ConfiguredTargetKeyCodec implements ObjectCodec { @Override @@ -424,6 +349,7 @@ public void serialize( context.serialize(key.getLabel(), codedOut); context.serialize(key.getConfigurationKey(), codedOut); context.serialize(key.getExecutionPlatformLabel(), codedOut); + context.serialize(key.shouldApplyRuleTransition(), codedOut); } @Override @@ -433,6 +359,7 @@ public ConfiguredTargetKey deserialize(DeserializationContext context, CodedInpu .setLabel(context.deserialize(codedIn)) .setConfigurationKey(context.deserialize(codedIn)) .setExecutionPlatformLabel(context.deserialize(codedIn)) + .setShouldApplyRuleTransition(context.deserialize(codedIn)) .build(); } } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java index 5a74f4933f205b..5a949309c04f1e 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java @@ -88,7 +88,7 @@ import com.google.devtools.build.lib.analysis.ConfiguredTargetValue; import com.google.devtools.build.lib.analysis.Dependency; import com.google.devtools.build.lib.analysis.DependencyKind; -import com.google.devtools.build.lib.analysis.InvalidVisibilityDependencyException; +import com.google.devtools.build.lib.analysis.InconsistentNullConfigException; import com.google.devtools.build.lib.analysis.PlatformOptions; import com.google.devtools.build.lib.analysis.TargetAndConfiguration; import com.google.devtools.build.lib.analysis.TargetConfiguredEvent; @@ -3897,8 +3897,7 @@ public void acceptConfiguredTargetAndData(ConfiguredTargetAndData value, int ind public void acceptConfiguredTargetAndDataError(ConfiguredValueCreationException error) {} @Override - public void acceptConfiguredTargetAndDataError( - InvalidVisibilityDependencyException error) {} + public void acceptConfiguredTargetAndDataError(InconsistentNullConfigException error) {} }; EvaluationResult result; @@ -3910,7 +3909,7 @@ public void acceptConfiguredTargetAndDataError( .setLabel(label) .setConfiguration(configuration) .build(), - /* transitionKey= */ null, + /* transitionKeys= */ ImmutableList.of(), TransitiveDependencyState.createForTesting(), sink, /* outputIndex= */ 0), diff --git a/src/test/java/com/google/devtools/build/lib/analysis/BUILD b/src/test/java/com/google/devtools/build/lib/analysis/BUILD index aa6e0028a31696..9c4dff27ffa6a8 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/BUILD +++ b/src/test/java/com/google/devtools/build/lib/analysis/BUILD @@ -101,7 +101,6 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:config/transitions/transition_collector", "//src/main/java/com/google/devtools/build/lib/analysis:config/transitions/transition_factory", "//src/main/java/com/google/devtools/build/lib/analysis:configured_target", - "//src/main/java/com/google/devtools/build/lib/analysis:configured_target_value", "//src/main/java/com/google/devtools/build/lib/analysis:dependency", "//src/main/java/com/google/devtools/build/lib/analysis:dependency_key", "//src/main/java/com/google/devtools/build/lib/analysis:dependency_kind", diff --git a/src/test/java/com/google/devtools/build/lib/analysis/ConfigurableAttributesTest.java b/src/test/java/com/google/devtools/build/lib/analysis/ConfigurableAttributesTest.java index 280593f8edbf3d..ef6be77deebae6 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/ConfigurableAttributesTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/ConfigurableAttributesTest.java @@ -13,7 +13,6 @@ // limitations under the License. package com.google.devtools.build.lib.analysis; -import static com.google.common.testing.GcFinalization.awaitClear; import static com.google.common.truth.Truth.assertThat; import static com.google.devtools.build.lib.packages.Attribute.attr; import static com.google.devtools.build.lib.packages.BuildType.LABEL_LIST; @@ -33,12 +32,9 @@ import com.google.devtools.build.lib.packages.RuleClass.ToolchainResolutionMode; import com.google.devtools.build.lib.packages.Type; import com.google.devtools.build.lib.skyframe.ConfiguredTargetAndData; -import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey; -import com.google.devtools.build.lib.skyframe.SkyframeExecutorWrappingWalkableGraph; import com.google.devtools.build.lib.testutil.TestRuleClassProvider; import com.google.devtools.build.lib.util.FileTypeSet; import java.io.IOException; -import java.lang.ref.WeakReference; import java.util.Collection; import java.util.Set; import org.junit.Before; @@ -1931,71 +1927,4 @@ public void selectWithLabelKeysInMacro() throws Exception { /*expected:*/ ImmutableList.of("bin java/foo/libb.jar", "bin java/foo/libb2.jar"), /*not expected:*/ ImmutableList.of("bin java/foo/liba.jar", "bin java/foo/liba2.jar")); } - - @Test - public void proxyKeysAreRetained() throws Exception { - // This test case verifies that when a ProxyConfiguredTargetKey is created, it is retained. - scratch.file( - "conditions/BUILD", - "constraint_setting(name = 'animal')", - "constraint_value(name = 'manatee', constraint_setting = 'animal')", - "constraint_value(name = 'koala', constraint_setting = 'animal')", - "platform(", - " name = 'manatee_platform',", - " constraint_values = [':manatee'],", - ")", - "platform(", - " name = 'koala_platform',", - " constraint_values = [':koala'],", - ")"); - scratch.file( - "check/BUILD", - "filegroup(name = 'adep', srcs = ['afile'])", - "filegroup(name = 'bdep', srcs = ['bfile'])", - "filegroup(name = 'hello',", - " srcs = select({", - " '//conditions:manatee': [':adep'],", - " '//conditions:koala': [':bdep'],", - " }))"); - - useConfiguration("--experimental_platforms=//conditions:manatee_platform"); - ConfiguredTarget hello = getConfiguredTarget("//check:hello"); - - var koalaLabel = Label.parseCanonical("//conditions:koala"); - - // Shakes the interner to try to get any non-strongly reachable keys to fall out. This should - // cause the ProxyConfiguredTargetKey created for "//conditions:koala" to fall out if it's not - // otherwise retained. - // - // Creates and inserts a canary key into the interner that can be used to detect eviction of - // weak keys. - var canaryKey = new WeakReference<>(ConfiguredTargetKey.builder().setLabel(koalaLabel).build()); - awaitClear(canaryKey); - // Once we get here we know that the canaryKey is no longer in the weak interner. Due to the - // collection properties of weak references, that implies the interner now has no weakly - // reachable keys at all. - - // Since //conditions:koala is a ConfigCondition, so it would be requested by //check:hello - // using //check:hello's configuration. - var koalaOwner = - ConfiguredTargetKey.builder() - .setLabel(koalaLabel) - .setConfigurationKey(hello.getConfigurationKey()) - .build(); - // Uses a WalkableGraph lookup to ensure there is an existing //conditions:koala instance that - // was created using koalaOwner. - var walkableGraph = SkyframeExecutorWrappingWalkableGraph.of(skyframeExecutor); - var koala = (ConfiguredTargetValue) walkableGraph.getValue(koalaOwner.toKey()); - assertThat(koala).isNotNull(); - - // constraint_value has a NoConfigTransition rule transition so a corresponding proxy key - // should exist. - ConfiguredTargetKey koalaKey = - ConfiguredTargetKey.builder() - .setLabel(koalaLabel) - .setConfigurationKey(koala.getConfiguredTarget().getConfigurationKey()) - .build(); - assertThat(koalaKey.isProxy()).isTrue(); - assertThat(koalaKey.toKey()).isEqualTo(koalaOwner); - } } diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewForTesting.java b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewForTesting.java index 91e7395e9688ff..8b2ad5419fb708 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewForTesting.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewForTesting.java @@ -425,7 +425,7 @@ private OrderedSetMultimap getPrerequis eventHandler, new PrerequisiteParameters( ConfiguredTargetKey.fromConfiguredTarget(target), - state.targetAndConfiguration.getTarget().getAssociatedRule(), + state.targetAndConfiguration.getTarget(), /* aspects= */ ImmutableList.of(), skyframeBuildView.getStarlarkTransitionCache(), toolchainContexts, diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKeyTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKeyTest.java index c44c5b4b2d3e80..1201693bf13ef4 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKeyTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKeyTest.java @@ -13,111 +13,57 @@ // limitations under the License. package com.google.devtools.build.lib.skyframe; -import static com.google.common.truth.Truth.assertThat; - -import com.google.common.collect.ImmutableList; -import com.google.devtools.build.lib.analysis.config.BuildOptions; import com.google.devtools.build.lib.analysis.config.BuildOptions.MapBackedChecksumCache; import com.google.devtools.build.lib.analysis.config.BuildOptions.OptionsChecksumCache; -import com.google.devtools.build.lib.analysis.config.CoreOptions; import com.google.devtools.build.lib.analysis.util.BuildViewTestCase; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.skyframe.serialization.testutils.SerializationTester; -import com.google.testing.junit.testparameterinjector.TestParameter; import com.google.testing.junit.testparameterinjector.TestParameterInjector; -import java.util.concurrent.atomic.AtomicInteger; import org.junit.Test; import org.junit.runner.RunWith; @RunWith(TestParameterInjector.class) public final class ConfiguredTargetKeyTest extends BuildViewTestCase { - private static final AtomicInteger nextId = new AtomicInteger(); - - @Test - public void testDelegation( - @TestParameter boolean useNullConfig, @TestParameter boolean isToolchainKey) { - var baseKey = createKey(useNullConfig, isToolchainKey); - - assertThat(baseKey.isProxy()).isFalse(); - assertThat(baseKey.toKey()).isSameInstanceAs(baseKey); - - BuildConfigurationKey newConfigurationKey = getNewUniqueConfigurationKey(); - var delegatingKey = - ConfiguredTargetKey.builder() - .setDelegate(baseKey) - .setConfigurationKey(newConfigurationKey) - .build(); - assertThat(delegatingKey.isProxy()).isTrue(); - assertThat(delegatingKey.toKey()).isSameInstanceAs(baseKey); - assertThat(delegatingKey.getLabel()).isSameInstanceAs(baseKey.getLabel()); - assertThat(delegatingKey.getConfigurationKey()).isSameInstanceAs(newConfigurationKey); - assertThat(delegatingKey.getExecutionPlatformLabel()) - .isSameInstanceAs(baseKey.getExecutionPlatformLabel()); - - // Building a key with the same parameters as the delegating key returns the delegating key. - var similarKey = - ConfiguredTargetKey.builder() - .setLabel(delegatingKey.getLabel()) - .setConfigurationKey(delegatingKey.getConfigurationKey()) - .setExecutionPlatformLabel(delegatingKey.getExecutionPlatformLabel()) - .build(); - assertThat(similarKey).isSameInstanceAs(delegatingKey); - } - - @Test - public void existingKey_inhibitsDelegation( - @TestParameter boolean useNullConfig, @TestParameter boolean isToolchainKey) { - var baseKey = createKey(useNullConfig, isToolchainKey); - - BuildConfigurationKey newConfigurationKey = getNewUniqueConfigurationKey(); - - var existingKey = - ConfiguredTargetKey.builder() - .setLabel(baseKey.getLabel()) - .setConfigurationKey(newConfigurationKey) - .setExecutionPlatformLabel(baseKey.getExecutionPlatformLabel()) - .build(); - - var delegatingKey = - ConfiguredTargetKey.builder() - .setDelegate(baseKey) - .setConfigurationKey(newConfigurationKey) - .build(); - - assertThat(delegatingKey).isSameInstanceAs(existingKey); - } - @Test public void testCodec() throws Exception { - var nullConfigKey = createKey(/* useNullConfig= */ true, /* isToolchainKey= */ false); - var keyWithConfig = createKey(/* useNullConfig= */ false, /* isToolchainKey= */ false); - var toolchainKey = createKey(/* useNullConfig= */ false, /* isToolchainKey= */ true); - - var delegatingToNullConfig = - ConfiguredTargetKey.builder() - .setDelegate(nullConfigKey) - .setConfigurationKey(targetConfigKey) - .build(); - var delegatingToKeyWithConfig = - ConfiguredTargetKey.builder().setDelegate(keyWithConfig).build(); - var delegatingToToolchainKey = - ConfiguredTargetKey.builder() - .setDelegate(toolchainKey) - .setConfigurationKey(getNewUniqueConfigurationKey()) - .build(); + var nullConfigKey = + createKey( + /* useNullConfig= */ true, + /* isToolchainKey= */ false, + /* shouldApplyRuleTransition= */ true); + var keyWithConfig = + createKey( + /* useNullConfig= */ false, + /* isToolchainKey= */ false, + /* shouldApplyRuleTransition= */ true); + var keyWithFinalConfig = + createKey( + /* useNullConfig= */ false, + /* isToolchainKey= */ false, + /* shouldApplyRuleTransition= */ false); + var toolchainKey = + createKey( + /* useNullConfig= */ false, + /* isToolchainKey= */ true, + /* shouldApplyRuleTransition= */ true); + var toolchainKeyWithFinalConfig = + createKey( + /* useNullConfig= */ false, + /* isToolchainKey= */ true, + /* shouldApplyRuleTransition= */ false); new SerializationTester( nullConfigKey, keyWithConfig, + keyWithFinalConfig, toolchainKey, - delegatingToNullConfig, - delegatingToKeyWithConfig, - delegatingToToolchainKey) + toolchainKeyWithFinalConfig) .addDependency(OptionsChecksumCache.class, new MapBackedChecksumCache()) .runTests(); } - private ConfiguredTargetKey createKey(boolean useNullConfig, boolean isToolchainKey) { + private ConfiguredTargetKey createKey( + boolean useNullConfig, boolean isToolchainKey, boolean shouldApplyRuleTransition) { var key = ConfiguredTargetKey.builder().setLabel(Label.parseCanonicalUnchecked("//p:key")); if (!useNullConfig) { key.setConfigurationKey(targetConfigKey); @@ -125,15 +71,7 @@ private ConfiguredTargetKey createKey(boolean useNullConfig, boolean isToolchain if (isToolchainKey) { key.setExecutionPlatformLabel(Label.parseCanonicalUnchecked("//platforms:b")); } + key.setShouldApplyRuleTransition(shouldApplyRuleTransition); return key.build(); } - - private BuildConfigurationKey getNewUniqueConfigurationKey() { - BuildOptions newOptions = targetConfigKey.getOptions().clone(); - var coreOptions = newOptions.get(CoreOptions.class); - coreOptions.affectedByStarlarkTransition = - ImmutableList.of("//fake:id" + nextId.getAndIncrement()); - assertThat(newOptions.checksum()).isNotEqualTo(targetConfigKey.getOptions().checksum()); - return BuildConfigurationKey.withoutPlatformMapping(newOptions); - } } From 67c31eedb189d0b320bd4b2a011f25e9c57548c6 Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 07:47:25 -0700 Subject: [PATCH 33/51] Implement `java_common.to_java_binary_info` in Starlark and delete corresponding native code PiperOrigin-RevId: 545668698 Change-Id: I1f57ce33b1c34759c82c1f348ed44f7bdd5b800a --- .../lib/rules/java/JavaStarlarkCommon.java | 45 ----------- .../starlarkbuildapi/java/JavaCommonApi.java | 13 ---- .../builtins_bzl/common/java/java_binary.bzl | 3 +- .../java_common_internal_for_builtins.bzl | 11 --- .../builtins_bzl/common/java/java_info.bzl | 77 ++++++++++++++++++- .../common/java/java_plugin_info.bzl | 4 +- 6 files changed, 79 insertions(+), 74 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java index 9dd07f9a19022d..4266567efaa752 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java @@ -44,7 +44,6 @@ import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; import com.google.devtools.build.lib.rules.cpp.CcInfo; import com.google.devtools.build.lib.rules.cpp.CppFileTypes; -import com.google.devtools.build.lib.rules.java.JavaRuleOutputJarsProvider.JavaOutput; import com.google.devtools.build.lib.starlarkbuildapi.core.ProviderApi; import com.google.devtools.build.lib.starlarkbuildapi.core.TransitiveInfoCollectionApi; import com.google.devtools.build.lib.starlarkbuildapi.java.JavaCommonApi; @@ -52,7 +51,6 @@ import java.util.LinkedHashSet; import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; import net.starlark.java.eval.EvalException; import net.starlark.java.eval.Module; import net.starlark.java.eval.Sequence; @@ -353,49 +351,6 @@ protected static void checkPrivateAccess(StarlarkThread thread) throws EvalExcep } } - @Override - public JavaInfo toJavaBinaryInfo(JavaInfo javaInfo, StarlarkThread thread) throws EvalException { - checkPrivateAccess(thread); - JavaRuleOutputJarsProvider ruleOutputs = - JavaRuleOutputJarsProvider.builder() - .addJavaOutput( - javaInfo.getJavaOutputs().stream() - .map( - output -> - JavaOutput.create( - output.getClassJar(), - null, - null, - output.getGeneratedClassJar(), - output.getGeneratedSourceJar(), - output.getNativeHeadersJar(), - output.getManifestProto(), - output.getJdeps(), - output.getSourceJars())) - .collect(Collectors.toList())) - .build(); - JavaInfo.Builder builder = JavaInfo.Builder.create(); - if (javaInfo.getProvider(JavaCompilationInfoProvider.class) != null) { - builder.javaCompilationInfo(javaInfo.getCompilationInfoProvider()); - } else if (javaInfo.getProvider(JavaCompilationArgsProvider.class) != null) { - JavaCompilationArgsProvider compilationArgsProvider = - javaInfo.getProvider(JavaCompilationArgsProvider.class); - builder.javaCompilationInfo( - new JavaCompilationInfoProvider.Builder() - .setCompilationClasspath(compilationArgsProvider.getTransitiveCompileTimeJars()) - .setRuntimeClasspath(compilationArgsProvider.getRuntimeJars()) - .build()); - } - if (javaInfo.getProvider(JavaGenJarsProvider.class) != null) { - builder.javaGenJars(javaInfo.getGenJarsProvider()); - } - return builder - .javaCcInfo(javaInfo.getProvider(JavaCcInfoProvider.class)) - .javaSourceJars(javaInfo.getProvider(JavaSourceJarsProvider.class)) - .javaRuleOutputs(ruleOutputs) - .build(); - } - @Override public Sequence getBuildInfo( StarlarkRuleContext starlarkRuleContext, boolean isStampingEnabled, StarlarkThread thread) diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java index e20653f6f1e5ae..700f54e0467800 100644 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java @@ -600,19 +600,6 @@ JavaInfoT setAnnotationProcessing( String getTargetKind(Object target, boolean dereferenceAliases, StarlarkThread thread) throws EvalException; - @StarlarkMethod( - name = "to_java_binary_info", - doc = "Returns a copy of the given JavaInfo with minimal info returned by a java_binary", - parameters = { - @Param( - name = "java_info", - positional = true, - named = false, - doc = "The JavaInfo to enhance."), - }, - useStarlarkThread = true) - JavaInfoT toJavaBinaryInfo(JavaInfoT javaInfo, StarlarkThread thread) throws EvalException; - @StarlarkMethod( name = "get_build_info", documented = false, diff --git a/src/main/starlark/builtins_bzl/common/java/java_binary.bzl b/src/main/starlark/builtins_bzl/common/java/java_binary.bzl index 1939d91f1fcc1b..ad8d448b4b4002 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_binary.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_binary.bzl @@ -22,14 +22,13 @@ load(":common/cc/semantics.bzl", cc_semantics = "semantics") load(":common/proto/proto_info.bzl", "ProtoInfo") load(":common/cc/cc_info.bzl", "CcInfo") load(":common/paths.bzl", "paths") -load(":common/java/java_info.bzl", "JavaInfo") +load(":common/java/java_info.bzl", "JavaInfo", "to_java_binary_info") load(":common/java/java_plugin_info.bzl", "JavaPluginInfo") load(":common/java/java_common.bzl", "java_common") load( ":common/java/java_common_internal_for_builtins.bzl", "collect_native_deps_dirs", "get_runtime_classpath_for_archive", - "to_java_binary_info", ) CcLauncherInfo = _builtins.internal.cc_internal.launcher_provider diff --git a/src/main/starlark/builtins_bzl/common/java/java_common_internal_for_builtins.bzl b/src/main/starlark/builtins_bzl/common/java/java_common_internal_for_builtins.bzl index 3bca5c95308ecd..768e4ae97f4af4 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_common_internal_for_builtins.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_common_internal_for_builtins.bzl @@ -213,17 +213,6 @@ def target_kind(target, dereference_aliases = False): dereference_aliases = dereference_aliases, ) -def to_java_binary_info(java_info): - """Get a copy of the given JavaInfo with minimal info returned by a java_binary - - Args: - java_info: (JavaInfo) A JavaInfo provider instance - - Returns: - (JavaInfo) A JavaInfo instance representing a java_binary target - """ - return _java_common_internal.to_java_binary_info(java_info) - def get_build_info(ctx, is_stamping_enabled): """Get the artifacts representing the workspace status for this build diff --git a/src/main/starlark/builtins_bzl/common/java/java_info.bzl b/src/main/starlark/builtins_bzl/common/java/java_info.bzl index b5c091effc494a..078117ab83a4a2 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_info.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_info.bzl @@ -17,7 +17,7 @@ Definition of JavaInfo provider. """ load(":common/cc/cc_common.bzl", "cc_common") -load(":common/java/java_plugin_info.bzl", "merge_without_outputs") +load(":common/java/java_plugin_info.bzl", "EMPTY_PLUGIN_DATA", "merge_without_outputs") load(":common/cc/cc_info.bzl", "CcInfo") # TODO(hvd): remove this when: @@ -71,6 +71,81 @@ _JavaGenJarsInfo = provider( }, ) +_JavaCompilationInfo = provider( + doc = "Compilation information in Java rules, for perusal of aspects and tools.", + fields = { + "boot_classpath": "Boot classpath for this Java target.", + "javac_options": "Options to the java compiler.", + "compilation_classpath": "Compilation classpath for this Java target.", + "runtime_classpath": "Run-time classpath for this Java target.", + }, +) + +def to_java_binary_info(java_info): + """Get a copy of the given JavaInfo with minimal info returned by a java_binary + + Args: + java_info: (JavaInfo) A JavaInfo provider instance + + Returns: + (JavaInfo) A JavaInfo instance representing a java_binary target + """ + result = { + "transitive_runtime_jars": depset(), + "transitive_runtime_deps": depset(), # deprecated + "transitive_compile_time_jars": depset(), + "transitive_deps": depset(), # deprecated + "compile_jars": depset(), + "full_compile_jars": depset(), + "_transitive_full_compile_time_jars": depset(), + "_compile_time_java_dependencies": depset(), + "runtime_output_jars": [], + "plugins": EMPTY_PLUGIN_DATA, + "api_generating_plugins": EMPTY_PLUGIN_DATA, + "module_flags_info": _ModuleFlagsInfo(add_exports = depset(), add_opens = depset()), + "_neverlink": False, + "_constraints": [], + "annotation_processing": java_info.annotation_processing, + "cc_link_params_info": getattr(java_info, "cc_link_params_info", None), + "transitive_native_libraries": java_info.transitive_native_libraries, + "source_jars": java_info.source_jars, + "transitive_source_jars": java_info.transitive_source_jars, + } + + compilation_info = None + if hasattr(java_info, "compilation_info"): + compilation_info = java_info.compilation_info + elif java_info.transitive_compile_time_jars or java_info.transitive_runtime_jars: + compilation_info = _JavaCompilationInfo( + boot_classpath = None, + javac_options = [], + compilation_classpath = java_info.transitive_compile_time_jars, + runtime_classpath = java_info.transitive_runtime_jars, + ) + result["compilation_info"] = compilation_info + + java_outputs = [ + _JavaOutputInfo( + compile_jar = None, + compile_jdeps = None, + class_jar = output.class_jar, + ijar = output.compile_jar, # deprecated + generated_class_jar = output.generated_class_jar, + generated_source_jar = output.generated_source_jar, + native_headers_jar = output.native_headers_jar, + manifest_proto = output.manifest_proto, + jdeps = output.jdeps, + source_jars = output.source_jars, + source_jar = output.source_jar, # deprecated + ) + for output in java_info.java_outputs + ] + result.update( + java_outputs = java_outputs, + outputs = _JavaRuleOutputJarsInfo(jars = java_outputs, jdeps = None, native_headers = None), + ) + return _new_javainfo(**result) + def _validate_provider_list(provider_list, what, expected_provider_type): _java_common_internal.check_provider_instances(provider_list, what, expected_provider_type) diff --git a/src/main/starlark/builtins_bzl/common/java/java_plugin_info.bzl b/src/main/starlark/builtins_bzl/common/java/java_plugin_info.bzl index 8743af38e8edda..23502f9e08dabd 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_plugin_info.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_plugin_info.bzl @@ -27,7 +27,7 @@ _JavaPluginDataInfo = provider( }, ) -_EMPTY_PLUGIN_DATA = _JavaPluginDataInfo( +EMPTY_PLUGIN_DATA = _JavaPluginDataInfo( processor_classes = depset(), processor_jars = depset(), processor_data = depset(), @@ -69,7 +69,7 @@ def _javaplugininfo_init( ) return { "plugins": plugins, - "api_generating_plugins": plugins if generates_api else _EMPTY_PLUGIN_DATA, + "api_generating_plugins": plugins if generates_api else EMPTY_PLUGIN_DATA, "java_outputs": java_infos.java_outputs, } From 42ced78aef55654cf11af61f9f1a2ff0879d974c Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 08:09:49 -0700 Subject: [PATCH 34/51] In `ResourceJarActionBuilder`, when all resource artifacts use the default path, discard the `PathFragment` objects. The default path for resources can be computed using a lazy transform and does not need to be retained in the command line. PiperOrigin-RevId: 545675732 Change-Id: I356087bd3a9c2bb99443f3fd5e8c96a378564134 --- .../rules/java/ResourceJarActionBuilder.java | 60 +++++++++++++------ 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/ResourceJarActionBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/java/ResourceJarActionBuilder.java index 61a7f4b274bc11..d13b52da2009a0 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/ResourceJarActionBuilder.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/ResourceJarActionBuilder.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.ParamFileInfo; import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; @@ -29,7 +30,6 @@ import com.google.devtools.build.lib.collect.nestedset.Order; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.errorprone.annotations.CanIgnoreReturnValue; -import java.util.List; import java.util.Map; /** Builds the action to package the resources for a Java rule into a jar. */ @@ -43,7 +43,7 @@ public class ResourceJarActionBuilder { private Map resources = ImmutableMap.of(); private NestedSet resourceJars = NestedSetBuilder.emptySet(Order.STABLE_ORDER); private ImmutableList classpathResources = ImmutableList.of(); - private List messages = ImmutableList.of(); + private ImmutableList messages = ImmutableList.of(); private JavaToolchainProvider javaToolchain; private NestedSet additionalInputs = NestedSetBuilder.emptySet(Order.STABLE_ORDER); @@ -105,16 +105,7 @@ public void build(JavaSemantics semantics, RuleContext ruleContext, String execG if (!resourceJars.isEmpty()) { command.addExecPaths("--sources", resourceJars); } - if (!resources.isEmpty() || !messages.isEmpty()) { - command.add("--resources"); - for (Map.Entry resource : resources.entrySet()) { - addAsResourcePrefixedExecPath(resource.getKey(), resource.getValue(), command); - } - for (Artifact message : messages) { - addAsResourcePrefixedExecPath( - semantics.getDefaultJavaResourcePath(message.getRootRelativePath()), message, command); - } - } + addResources(command, semantics); if (!classpathResources.isEmpty()) { command.addExecPaths("--classpath_resources", classpathResources); } @@ -136,13 +127,46 @@ public void build(JavaSemantics semantics, RuleContext ruleContext, String execG .build(ruleContext)); } - private static void addAsResourcePrefixedExecPath( - PathFragment resourcePath, Artifact artifact, CustomCommandLine.Builder builder) { - PathFragment execPath = artifact.getExecPath(); - if (execPath.equals(resourcePath)) { - builder.addFormatted("%s", resourcePath); + private void addResources(CustomCommandLine.Builder command, JavaSemantics semantics) { + if (resources.isEmpty() && messages.isEmpty()) { + return; + } + + command.add("--resources"); + ImmutableList resourcesWithDefaultPath; + + // When all resources use the default path (common case), save memory by throwing away those + // path fragments. The artifacts can be lazily converted to default-prefixed strings. + if (resources.entrySet().stream() + .allMatch(e -> e.getKey().equals(defaultResourcePath(e.getValue(), semantics)))) { + resourcesWithDefaultPath = + ImmutableList.builderWithExpectedSize(resources.size() + messages.size()) + .addAll(resources.values()) + .addAll(messages) + .build(); } else { - builder.addFormatted("%s:%s", execPath, resourcePath); + command.addObject( + Lists.transform( + ImmutableList.copyOf(resources.entrySet()), + e -> resourcePrefixedExecPath(e.getKey(), e.getValue()))); + resourcesWithDefaultPath = messages; + } + + if (!resourcesWithDefaultPath.isEmpty()) { + command.addObject( + Lists.transform( + resourcesWithDefaultPath, + artifact -> + resourcePrefixedExecPath(defaultResourcePath(artifact, semantics), artifact))); } } + + private static PathFragment defaultResourcePath(Artifact artifact, JavaSemantics semantics) { + return semantics.getDefaultJavaResourcePath(artifact.getRootRelativePath()); + } + + private static String resourcePrefixedExecPath(PathFragment resourcePath, Artifact artifact) { + PathFragment execPath = artifact.getExecPath(); + return execPath.equals(resourcePath) ? execPath.getPathString() : execPath + ":" + resourcePath; + } } From f22d35ffbe02194c2380f6e27b27293d62bcf799 Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 08:48:56 -0700 Subject: [PATCH 35/51] Delete top-level rule transition special case code. This is no longer needed now that configured targets perform their own rule transitions. * Decomposes configuration logging differently. The BuildConfigurationValue is often not available where the transitions occur, sometimes never being computed at all, as is the case of an attribute transition followed immediately with a rule transition. In the case of a child-side rule transition, the BuildConfigurationValue isn't computed locally. It's computed instead by the delegate, which doesn't know the parent checksum. To work around this, logs ConfigurationTransitionEvents as a pair of parent and child hashes, then logs ConfigurationValueEvents as they are created in BuildConfigurationFunction. This data can be joined in InvocationLogger to recover what used to be in the ConfigRequestedEvents. * There were a couple of places where additional filtering was needed, similar to filters added for aquery/cquery. SequencedSkyframeExecutor has a couple dump methods where nodes that only do delegation need to be filtered out. The same filter is also needed in ActionLookupValuesTraversal.accumulate. PiperOrigin-RevId: 545687484 Change-Id: I42060c6f399870ee0abd08dffa1364467f2a4098 --- .../build/lib/analysis/AnalysisUtils.java | 70 --------- .../google/devtools/build/lib/analysis/BUILD | 39 ++--- .../build/lib/analysis/BuildView.java | 77 +++------- .../lib/analysis/ConfigurationsCollector.java | 35 ----- .../lib/analysis/ConfigurationsResult.java | 67 -------- .../config/ConfigurationResolver.java | 144 +----------------- .../config/ConfigurationTransitionEvent.java | 39 +++++ .../config/ConfigurationValueEvent.java | 27 ++++ .../build/lib/analysis/producers/BUILD | 1 + .../producers/DependencyProducer.java | 10 ++ .../TargetAndConfigurationProducer.java | 8 +- .../AnalysisAndExecutionPhaseRunner.java | 5 +- .../lib/buildtool/AnalysisPhaseRunner.java | 5 +- .../skyframe/ActionLookupValuesTraversal.java | 17 ++- .../google/devtools/build/lib/skyframe/BUILD | 3 +- .../skyframe/BuildConfigurationFunction.java | 23 +-- .../skyframe/ConfiguredTargetFunction.java | 7 +- .../skyframe/SequencedSkyframeExecutor.java | 12 +- .../build/lib/skyframe/SkyframeExecutor.java | 132 +--------------- .../lib/analysis/AnalysisCachingTest.java | 43 +++--- .../AnalysisFailureReportingTest.java | 33 ++-- .../google/devtools/build/lib/analysis/BUILD | 2 + .../lib/analysis/TargetCompleteEventTest.java | 6 +- .../test/TrimTestConfigurationTest.java | 48 ++++-- .../analysis/util/BuildViewForTesting.java | 23 --- .../lib/analysis/util/BuildViewTestCase.java | 8 +- .../starlark_configurations_test.sh | 2 +- 27 files changed, 259 insertions(+), 627 deletions(-) delete mode 100644 src/main/java/com/google/devtools/build/lib/analysis/ConfigurationsCollector.java delete mode 100644 src/main/java/com/google/devtools/build/lib/analysis/ConfigurationsResult.java create mode 100644 src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationTransitionEvent.java create mode 100644 src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationValueEvent.java diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisUtils.java b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisUtils.java index ff77cf43df8f07..9166a049afae67 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisUtils.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisUtils.java @@ -11,36 +11,20 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package com.google.devtools.build.lib.analysis; -import static com.google.common.collect.ImmutableSet.toImmutableSet; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; -import com.google.common.collect.Multimap; import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; -import com.google.devtools.build.lib.analysis.config.ConfigurationResolver; -import com.google.devtools.build.lib.analysis.config.ConfigurationResolver.TopLevelTargetsAndConfigsResult; -import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; -import com.google.devtools.build.lib.analysis.config.TransitionResolver; -import com.google.devtools.build.lib.analysis.config.transitions.ConfigurationTransition; -import com.google.devtools.build.lib.analysis.config.transitions.NoTransition; import com.google.devtools.build.lib.cmdline.Label; -import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.devtools.build.lib.packages.BuildType; import com.google.devtools.build.lib.packages.BuiltinProvider; import com.google.devtools.build.lib.packages.Info; import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; import com.google.devtools.build.lib.packages.StarlarkProviderWrapper; -import com.google.devtools.build.lib.packages.Target; import com.google.devtools.build.lib.packages.TriState; import com.google.devtools.build.lib.packages.Type; import com.google.devtools.build.lib.vfs.PathFragment; -import java.util.Collection; import java.util.List; /** @@ -190,58 +174,4 @@ public static void checkProvider(Class cla clazz + " is generated by @AutoValue; use " + clazz.getSuperclass() + " instead"); } } - - /** - * Given a set of *top-level* targets and a configuration collection, evaluate top level - * transitions, resolve configurations and return the appropriate pair for - * each target. - * - *

Preserves the original input ordering. - */ - public static TopLevelTargetsAndConfigsResult getTargetsWithConfigs( - BuildConfigurationValue targetConfiguration, - Collection targets, - ExtendedEventHandler eventHandler, - ConfiguredRuleClassProvider ruleClassProvider, - ConfigurationsCollector configurationsCollector) - throws InvalidConfigurationException, InterruptedException { - // We use a set here to remove duplicate nodes; this can happen for input files and package - // groups. - ImmutableSet nodes = - targets.stream() - .map(target -> new TargetAndConfiguration(target, targetConfiguration)) - .collect(toImmutableSet()); - - // We'll get the configs from ConfigurationsCollector#getConfigurations, which gets - // configurations for deps including transitions. - Multimap asDeps = - targetsToDeps(nodes, ruleClassProvider); - - return ConfigurationResolver.getConfigurationsFromExecutor( - nodes, asDeps, eventHandler, configurationsCollector); - } - - @VisibleForTesting - public static Multimap targetsToDeps( - Collection nodes, ConfiguredRuleClassProvider ruleClassProvider) { - Multimap asDeps = ArrayListMultimap.create(); - for (TargetAndConfiguration targetAndConfig : nodes) { - ConfigurationTransition transition = - TransitionResolver.evaluateTransition( - targetAndConfig.getConfiguration(), - NoTransition.INSTANCE, - targetAndConfig.getTarget(), - ruleClassProvider.getTrimmingTransitionFactory()); - if (targetAndConfig.getConfiguration() != null) { - // TODO(bazel-team): support top-level aspects - asDeps.put( - targetAndConfig.getConfiguration(), - DependencyKey.builder() - .setLabel(targetAndConfig.getLabel()) - .setTransition(transition) - .build()); - } - } - return asDeps; - } } diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BUILD b/src/main/java/com/google/devtools/build/lib/analysis/BUILD index a7ff85cb5efa80..e52884dbc244a5 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/BUILD +++ b/src/main/java/com/google/devtools/build/lib/analysis/BUILD @@ -82,7 +82,6 @@ java_library( ":config/transitions/patch_transition", ":config/transitions/split_transition", ":config/transitions/transition_factory", - ":configurations_collector", ":configured_object_value", ":configured_target", ":constraints/constraint_constants", @@ -290,6 +289,7 @@ java_library( ":config/build_options", ":config/config_conditions", ":config/config_matching_provider", + ":config/configuration_transition_event", ":config/core_options", ":config/execution_transition_factory", ":config/feature_set", @@ -314,7 +314,6 @@ java_library( ":config/transitions/starlark_exposed_rule_transition_factory", ":config/transitions/transition_collector", ":config/transitions/transition_factory", - ":configurations_collector", ":configured_target", ":constraints/constraint_constants", ":constraints/constraint_semantics", @@ -625,7 +624,6 @@ java_library( ":constraints/top_level_constraint_semantics", ":extra_action_artifacts_provider", ":make_environment_event", - ":target_and_configuration", ":test/coverage_report_action_factory", ":test/instrumented_files_info", ":top_level_artifact_context", @@ -673,22 +671,6 @@ java_library( ], ) -java_library( - name = "configurations_collector", - srcs = [ - "ConfigurationsCollector.java", - "ConfigurationsResult.java", - ], - deps = [ - ":config/build_configuration", - ":config/build_options", - ":config/invalid_configuration_exception", - ":dependency_key", - "//src/main/java/com/google/devtools/build/lib/events", - "//third_party:guava", - ], -) - java_library( name = "configured_object_value", srcs = ["ConfiguredObjectValue.java"], @@ -1817,6 +1799,25 @@ java_library( ], ) +java_library( + name = "config/configuration_transition_event", + srcs = ["config/ConfigurationTransitionEvent.java"], + deps = [ + "//src/main/java/com/google/devtools/build/lib/events", + "//third_party:auto_value", + ], +) + +java_library( + name = "config/configuration_value_event", + srcs = ["config/ConfigurationValueEvent.java"], + deps = [ + ":config/build_configuration", + "//src/main/java/com/google/devtools/build/lib/events", + "//third_party:auto_value", + ], +) + java_library( name = "config/core_option_converters", srcs = ["config/CoreOptionConverters.java"], diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java b/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java index 4407995f704bce..7407c3a31a240d 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java @@ -11,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package com.google.devtools.build.lib.analysis; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -39,7 +38,6 @@ import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; import com.google.devtools.build.lib.analysis.config.BuildOptions; import com.google.devtools.build.lib.analysis.config.ConfigRequestedEvent; -import com.google.devtools.build.lib.analysis.config.ConfigurationResolver.TopLevelTargetsAndConfigsResult; import com.google.devtools.build.lib.analysis.config.CoreOptions; import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; import com.google.devtools.build.lib.analysis.constraints.PlatformRestrictionsResult; @@ -72,7 +70,6 @@ import com.google.devtools.build.lib.pkgcache.PackageManager.PackageManagerStatistics; import com.google.devtools.build.lib.profiler.Profiler; import com.google.devtools.build.lib.profiler.SilentCloseable; -import com.google.devtools.build.lib.server.FailureDetails; import com.google.devtools.build.lib.server.FailureDetails.Analysis; import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; import com.google.devtools.build.lib.server.FailureDetails.TargetPatterns; @@ -246,7 +243,6 @@ public AnalysisResult update( // Prepare the analysis phase BuildConfigurationValue topLevelConfig; - TopLevelTargetsAndConfigsResult topLevelTargetsWithConfigsResult; // Configuration creation. // TODO(gregce): Consider dropping this phase and passing on-the-fly target / exec configs as // needed. This requires cleaning up the invalidation in SkyframeBuildView.setConfigurations. @@ -257,15 +253,6 @@ public AnalysisResult update( if (buildConfigurationsCreatedCallback != null) { buildConfigurationsCreatedCallback.run(topLevelConfig); } - try (SilentCloseable c = Profiler.instance().profile("AnalysisUtils.getTargetsWithConfigs")) { - topLevelTargetsWithConfigsResult = - AnalysisUtils.getTargetsWithConfigs( - topLevelConfig, - labelToTargetMap.values(), - eventHandler, - ruleClassProvider, - skyframeExecutor); - } skyframeBuildView.setConfiguration( eventHandler, topLevelConfig, viewOptions.maxConfigChangesToShow); @@ -273,24 +260,20 @@ public AnalysisResult update( eventBus.post(new MakeEnvironmentEvent(topLevelConfig.getMakeEnvironment())); eventBus.post(topLevelConfig.toBuildEvent()); - Collection topLevelTargetsWithConfigs = - topLevelTargetsWithConfigsResult.getTargetsAndConfigs(); - - for (TargetAndConfiguration pair : topLevelTargetsWithConfigs) { - if (pair.getConfiguration() != null && !pair.getConfiguration().equals(topLevelConfig)) { - // Log top-level rule transitioned configurations. - eventBus.post(new ConfigRequestedEvent(pair.getConfiguration(), topLevelConfig.checksum())); - } - } - + var configurationKey = topLevelConfig.getKey(); ImmutableList topLevelCtKeys = - topLevelTargetsWithConfigs.stream() - .map(BuildView::getConfiguredTargetKey) + labelToTargetMap.keySet().stream() + .map( + label -> + ConfiguredTargetKey.builder() + .setLabel(label) + .setConfigurationKey(configurationKey) + .build()) .collect(toImmutableList()); - ImmutableList aspectsKeys = + ImmutableList aspectKeys = createTopLevelAspectKeys( - aspects, aspectsParameters, topLevelTargetsWithConfigs, eventHandler); + aspects, aspectsParameters, labelToTargetMap.keySet(), topLevelConfig, eventHandler); getArtifactFactory().noteAnalysisStarting(); SkyframeAnalysisResult skyframeAnalysisResult; @@ -306,7 +289,7 @@ public AnalysisResult update( skyframeBuildView.analyzeAndExecuteTargets( eventHandler, topLevelCtKeys, - aspectsKeys, + aspectKeys, loadingResult.getTestsToRunLabels(), labelToTargetMap, topLevelOptions, @@ -334,7 +317,7 @@ public AnalysisResult update( eventHandler, labelToTargetMap, topLevelCtKeys, - aspectsKeys, + aspectKeys, topLevelOptions, eventBus, bugReporter, @@ -372,8 +355,7 @@ public AnalysisResult update( viewOptions, skyframeAnalysisResult, /* targetsToSkip= */ ImmutableSet.of(), - /* labelToTargetMap= */ labelToTargetMap, - topLevelTargetsWithConfigsResult.hasError(), + labelToTargetMap, /* includeExecutionPhase= */ true); } else { ImmutableSet targetsToSkip = ImmutableSet.of(); @@ -419,25 +401,17 @@ public AnalysisResult update( skyframeAnalysisResult, targetsToSkip, labelToTargetMap, - topLevelTargetsWithConfigsResult.hasError(), /* includeExecutionPhase= */ false); } logger.atInfo().log("Finished analysis"); return result; } - private static ConfiguredTargetKey getConfiguredTargetKey( - TargetAndConfiguration targetAndConfiguration) { - return ConfiguredTargetKey.builder() - .setLabel(targetAndConfiguration.getLabel()) - .setConfiguration(targetAndConfiguration.getConfiguration()) - .build(); - } - private ImmutableList createTopLevelAspectKeys( List aspects, ImmutableMap aspectsParameters, - Collection topLevelTargetsWithConfigs, + ImmutableSet

Skips targets with loading phase errors. - */ - ConfigurationsResult getConfigurations( - ExtendedEventHandler eventHandler, - BuildOptions fromOptions, - Iterable keys) - throws InvalidConfigurationException, InterruptedException; -} diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ConfigurationsResult.java b/src/main/java/com/google/devtools/build/lib/analysis/ConfigurationsResult.java deleted file mode 100644 index a50d2f74387a54..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/analysis/ConfigurationsResult.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2020 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.lib.analysis; - -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.ListMultimap; -import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; -import com.google.devtools.build.lib.analysis.config.BuildOptions; -import com.google.devtools.build.lib.events.ExtendedEventHandler; - -/** - * The result of {@link #getConfigurations(ExtendedEventHandler, BuildOptions, Iterable)} which also - * registers if an error was recorded. - */ -public class ConfigurationsResult { - private final ListMultimap configurations; - private final boolean hasError; - - private ConfigurationsResult( - ListMultimap configurations, - boolean hasError) { - this.configurations = configurations; - this.hasError = hasError; - } - - public boolean hasError() { - return hasError; - } - - public ListMultimap getConfigurationMap() { - return configurations; - } - - public static Builder newBuilder() { - return new Builder(); - } - - /** Builder for {@link ConfigurationsResult} */ - public static class Builder { - private final ListMultimap - configurations = ArrayListMultimap.create(); - private boolean hasError = false; - - public void put(BaseDependencySpecification key, BuildConfigurationValue value) { - configurations.put(key, value); - } - - public void setHasError() { - this.hasError = true; - } - - public ConfigurationsResult build() { - return new ConfigurationsResult(configurations, hasError); - } - } -} diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationResolver.java b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationResolver.java index 06501a54b32978..151956146f01e4 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationResolver.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationResolver.java @@ -11,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - package com.google.devtools.build.lib.analysis.config; import com.google.common.annotations.VisibleForTesting; @@ -19,10 +18,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Multimap; -import com.google.devtools.build.lib.analysis.BaseDependencySpecification; -import com.google.devtools.build.lib.analysis.ConfigurationsCollector; -import com.google.devtools.build.lib.analysis.ConfigurationsResult; import com.google.devtools.build.lib.analysis.Dependency; import com.google.devtools.build.lib.analysis.DependencyKey; import com.google.devtools.build.lib.analysis.DependencyKind; @@ -41,7 +36,6 @@ import com.google.devtools.build.lib.packages.Attribute; import com.google.devtools.build.lib.packages.AttributeTransitionData; import com.google.devtools.build.lib.packages.ConfiguredAttributeMapper; -import com.google.devtools.build.lib.packages.Target; import com.google.devtools.build.lib.skyframe.BuildConfigurationKey; import com.google.devtools.build.lib.skyframe.ConfiguredValueCreationException; import com.google.devtools.build.lib.skyframe.PlatformMappingValue; @@ -51,11 +45,8 @@ import com.google.devtools.build.skyframe.SkyframeLookupResult; import com.google.devtools.common.options.OptionsParsingException; import java.util.ArrayList; -import java.util.Collection; import java.util.Comparator; import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import javax.annotation.Nullable; @@ -194,8 +185,9 @@ public ImmutableList resolveConfiguration( .forEach( d -> eventHandler.post( - new ConfigRequestedEvent( - d.getConfiguration(), ctgValue.getConfiguration().checksum()))); + ConfigurationTransitionEvent.create( + ctgValue.getConfiguration().checksum(), + d.getConfiguration().checksum()))); } return ans; } @@ -339,38 +331,6 @@ private ImmutableList collectTransitionKeys( /** * Applies a configuration transition over a set of build options. * - *

This is only for callers that can't use {@link #applyTransitionWithSkyframe}. The difference - * is {@link #applyTransitionWithSkyframe} internally computes {@code details} with Skyframe, - * while this version requires it as a precomputed input. - * - *

prework - load all default values for reading build settings in Starlark transitions (by - * design, {@link BuildOptions} never holds default values of build settings) - * - *

postwork - replay events/throw errors from transition implementation function and validate - * the outputs of the transition. This only applies to Starlark transitions. - * - * @return the build options for the transitioned configuration. - */ - public static Map applyTransitionWithoutSkyframe( - BuildOptions fromOptions, - ConfigurationTransition transition, - StarlarkBuildSettingsDetailsValue details, - ExtendedEventHandler eventHandler, - StarlarkTransitionCache starlarkTransitionCache) - throws TransitionException, InterruptedException { - if (StarlarkTransition.doesStarlarkTransition(transition)) { - return starlarkTransitionCache.computeIfAbsent( - fromOptions, transition, details, eventHandler); - } - return transition.apply(TransitionUtil.restrict(transition, fromOptions), eventHandler); - } - - /** - * Applies a configuration transition over a set of build options. - * - *

Callers should use this over {@link #applyTransitionWithoutSkyframe}. Unlike that variation, - * this would may return null if it needs more Skyframe deps. - * *

postwork - replay events/throw errors from transition implementation function and validate * the outputs of the transition. This only applies to Starlark transitions. * @@ -415,102 +375,4 @@ private static StarlarkBuildSettingsDetailsValue getStarlarkBuildSettingsDetails StarlarkBuildSettingsDetailsValue.key(starlarkBuildSettings), TransitionException.class); } - - /** - * This method allows resolution of configurations outside of a skyfunction call. - * - *

Unlike {@link #resolveConfigurations}, this doesn't expect the current context to be - * evaluating dependencies of a parent target. So this method is also suitable for top-level - * targets. - * - *

Resolution consists of applying the per-target transitions specified in {@code - * targetsToEvaluate}. This can be used, e.g., to apply {@link - * com.google.devtools.build.lib.analysis.config.transitions.TransitionFactory}s over global - * top-level configurations. - * - *

Preserves the original input order (but merges duplicate nodes that might occur due to - * top-level configuration transitions) . Uses original (untrimmed, pre-transition) configurations - * for targets that can't be evaluated (e.g. due to loading phase errors). - * - *

This is suitable for feeding {@link - * com.google.devtools.build.lib.analysis.ConfiguredTargetValue} keys: as general principle {@link - * com.google.devtools.build.lib.analysis.ConfiguredTarget}s should have exactly as much - * information in their configurations as they need to evaluate and no more (e.g. there's no need - * for Android settings in a C++ configured target). - * - * @param defaultContext the original targets and starting configurations before applying rule - * transitions and trimming. When actual configurations can't be evaluated, these values are - * returned as defaults. See TODO below. - * @param targetsToEvaluate the inputs repackaged as dependencies, including rule-specific - * transitions - * @param eventHandler the error event handler - * @param configurationsCollector the collector which finds configurations for dependencies - */ - // TODO(bazel-team): error out early for targets that fail - failed configuration evaluations - // should never make it through analysis (and especially not seed ConfiguredTargetValues) - // TODO(gregce): merge this more with resolveConfigurations? One crucial difference is - // resolveConfigurations can null-return on missing deps since it executes inside Skyfunctions. - public static TopLevelTargetsAndConfigsResult getConfigurationsFromExecutor( - Iterable defaultContext, - Multimap targetsToEvaluate, - ExtendedEventHandler eventHandler, - ConfigurationsCollector configurationsCollector) - throws InvalidConfigurationException, InterruptedException { - - Map labelsToTargets = new HashMap<>(); - for (TargetAndConfiguration targetAndConfig : defaultContext) { - labelsToTargets.put(targetAndConfig.getLabel(), targetAndConfig.getTarget()); - } - - // Maps pairs to pairs for targets that - // could be successfully Skyframe-evaluated. - Map successfullyEvaluatedTargets = - new LinkedHashMap<>(); - boolean hasError = false; - if (!targetsToEvaluate.isEmpty()) { - for (BuildConfigurationValue fromConfig : targetsToEvaluate.keySet()) { - ConfigurationsResult configurationsResult = - configurationsCollector.getConfigurations( - eventHandler, fromConfig.getOptions(), targetsToEvaluate.get(fromConfig)); - hasError |= configurationsResult.hasError(); - for (Map.Entry evaluatedTarget : - configurationsResult.getConfigurationMap().entries()) { - Target target = labelsToTargets.get(evaluatedTarget.getKey().getLabel()); - successfullyEvaluatedTargets.put( - new TargetAndConfiguration(target, fromConfig), - new TargetAndConfiguration(target, evaluatedTarget.getValue())); - } - } - } - - LinkedHashSet result = new LinkedHashSet<>(); - for (TargetAndConfiguration originalInput : defaultContext) { - // If the configuration couldn't be determined (e.g. loading phase error), use the original. - result.add(successfullyEvaluatedTargets.getOrDefault(originalInput, originalInput)); - } - return new TopLevelTargetsAndConfigsResult(result, hasError); - } - - /** - * The result of {@link #getConfigurationsFromExecutor} which also registers if an error was - * recorded. - */ - public static class TopLevelTargetsAndConfigsResult { - private final Collection configurations; - private final boolean hasError; - - public TopLevelTargetsAndConfigsResult( - Collection configurations, boolean hasError) { - this.configurations = configurations; - this.hasError = hasError; - } - - public boolean hasError() { - return hasError; - } - - public Collection getTargetsAndConfigs() { - return configurations; - } - } } diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationTransitionEvent.java b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationTransitionEvent.java new file mode 100644 index 00000000000000..78ef62c82d2f44 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationTransitionEvent.java @@ -0,0 +1,39 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis.config; + +import com.google.auto.value.AutoValue; +import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; + +/** Posted when there is a configuration transition. */ +@AutoValue +public abstract class ConfigurationTransitionEvent + implements Comparable, Postable { + public static ConfigurationTransitionEvent create(String parentChecksum, String childChecksum) { + return new AutoValue_ConfigurationTransitionEvent(parentChecksum, childChecksum); + } + + public abstract String parentChecksum(); + + public abstract String childChecksum(); + + @Override + public final int compareTo(ConfigurationTransitionEvent that) { + int result = parentChecksum().compareTo(that.parentChecksum()); + if (result != 0) { + return result; + } + return childChecksum().compareTo(that.childChecksum()); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationValueEvent.java b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationValueEvent.java new file mode 100644 index 00000000000000..84f6c96434f6dc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/ConfigurationValueEvent.java @@ -0,0 +1,27 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis.config; + +import com.google.auto.value.AutoValue; +import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; + +/** Posted when a {@link BuildConfigurationValue} is created. */ +@AutoValue +public abstract class ConfigurationValueEvent implements Postable { + public static ConfigurationValueEvent create(BuildConfigurationValue configuration) { + return new AutoValue_ConfigurationValueEvent(configuration); + } + + public abstract BuildConfigurationValue configuration(); +} diff --git a/src/main/java/com/google/devtools/build/lib/analysis/producers/BUILD b/src/main/java/com/google/devtools/build/lib/analysis/producers/BUILD index 95ed8f1824bb75..a32c92ed850464 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/producers/BUILD +++ b/src/main/java/com/google/devtools/build/lib/analysis/producers/BUILD @@ -22,6 +22,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:config/build_options", "//src/main/java/com/google/devtools/build/lib/analysis:config/config_conditions", "//src/main/java/com/google/devtools/build/lib/analysis:config/config_matching_provider", + "//src/main/java/com/google/devtools/build/lib/analysis:config/configuration_transition_event", "//src/main/java/com/google/devtools/build/lib/analysis:config/invalid_configuration_exception", "//src/main/java/com/google/devtools/build/lib/analysis:config/starlark_transition_cache", "//src/main/java/com/google/devtools/build/lib/analysis:config/transitions/composing_transition", diff --git a/src/main/java/com/google/devtools/build/lib/analysis/producers/DependencyProducer.java b/src/main/java/com/google/devtools/build/lib/analysis/producers/DependencyProducer.java index bc62b56c18507a..e8610c8f2afabd 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/producers/DependencyProducer.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/producers/DependencyProducer.java @@ -28,6 +28,7 @@ import com.google.devtools.build.lib.analysis.DependencyResolver.ExecutionPlatformResult; import com.google.devtools.build.lib.analysis.InvalidVisibilityDependencyException; import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; +import com.google.devtools.build.lib.analysis.config.ConfigurationTransitionEvent; import com.google.devtools.build.lib.analysis.config.DependencyEvaluationException; import com.google.devtools.build.lib.analysis.starlark.StarlarkTransition.TransitionException; import com.google.devtools.build.lib.causes.LoadingFailedCause; @@ -206,6 +207,14 @@ private StateMachine processTransitionResult(Tasks tasks, ExtendedEventHandler l /* executionPlatformLabel= */ null); } + String parentChecksum = parameters.configurationKey().getOptionsChecksum(); + for (BuildConfigurationKey configuration : transitionedConfigurations.values()) { + String childChecksum = configuration.getOptionsChecksum(); + if (!parentChecksum.equals(childChecksum)) { + listener.post(ConfigurationTransitionEvent.create(parentChecksum, childChecksum)); + } + } + if (transitionedConfigurations.size() == 1) { BuildConfigurationKey patchedConfiguration = transitionedConfigurations.get(PATCH_TRANSITION_KEY); @@ -216,6 +225,7 @@ private StateMachine processTransitionResult(Tasks tasks, ExtendedEventHandler l /* executionPlatformLabel= */ null); } } + return computePrerequisites( AttributeConfiguration.ofSplit(transitionedConfigurations), /* executionPlatformLabel= */ null); diff --git a/src/main/java/com/google/devtools/build/lib/analysis/producers/TargetAndConfigurationProducer.java b/src/main/java/com/google/devtools/build/lib/analysis/producers/TargetAndConfigurationProducer.java index de3b1d8e52f28e..474fa84bbf03d0 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/producers/TargetAndConfigurationProducer.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/producers/TargetAndConfigurationProducer.java @@ -25,6 +25,7 @@ import com.google.devtools.build.lib.analysis.TargetAndConfiguration; import com.google.devtools.build.lib.analysis.TransitiveDependencyState; import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; +import com.google.devtools.build.lib.analysis.config.ConfigurationTransitionEvent; import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; import com.google.devtools.build.lib.analysis.config.StarlarkTransitionCache; import com.google.devtools.build.lib.analysis.config.transitions.ComposingTransition; @@ -282,12 +283,17 @@ private StateMachine processTransitionedKey(Tasks tasks, ExtendedEventHandler li return DONE; // There was an error. } - if (configurationKey.equals(preRuleTransitionKey.getConfigurationKey())) { + BuildConfigurationKey parentConfiguration = preRuleTransitionKey.getConfigurationKey(); + if (configurationKey.equals(parentConfiguration)) { // This key owns the configuration and the computation completes normally. lookUpConfigurationValue(tasks); return DONE; } + listener.post( + ConfigurationTransitionEvent.create( + parentConfiguration.getOptionsChecksum(), configurationKey.getOptionsChecksum())); + return new IdempotencyChecker(); } diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/AnalysisAndExecutionPhaseRunner.java b/src/main/java/com/google/devtools/build/lib/buildtool/AnalysisAndExecutionPhaseRunner.java index 21f81af7bbb886..9210adb9e7c842 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/AnalysisAndExecutionPhaseRunner.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/AnalysisAndExecutionPhaseRunner.java @@ -161,10 +161,7 @@ static AnalysisAndExecutionResult execute( env.getReporter().post(new NoAnalyzeEvent()); logger.atInfo().log("No analysis requested, so finished"); FailureDetail failureDetail = - BuildView.createAnalysisFailureDetail( - loadingResult, - /* skyframeAnalysisResult= */ null, - /* hasTopLevelTargetsWithConfigurationError= */ false); + BuildView.createAnalysisFailureDetail(loadingResult, /* skyframeAnalysisResult= */ null); if (failureDetail != null) { throw new BuildFailedException( failureDetail.getMessage(), DetailedExitCode.of(failureDetail)); diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/AnalysisPhaseRunner.java b/src/main/java/com/google/devtools/build/lib/buildtool/AnalysisPhaseRunner.java index a96bb2f6621329..6e8fcda45e19cf 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/AnalysisPhaseRunner.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/AnalysisPhaseRunner.java @@ -161,10 +161,7 @@ public static AnalysisResult execute( env.getReporter().post(new NoAnalyzeEvent()); logger.atInfo().log("No analysis requested, so finished"); FailureDetail failureDetail = - BuildView.createAnalysisFailureDetail( - loadingResult, - /* skyframeAnalysisResult= */ null, - /* hasTopLevelTargetsWithConfigurationError= */ false); + BuildView.createAnalysisFailureDetail(loadingResult, /* skyframeAnalysisResult= */ null); if (failureDetail != null) { throw new BuildFailedException( failureDetail.getMessage(), DetailedExitCode.of(failureDetail)); diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ActionLookupValuesTraversal.java b/src/main/java/com/google/devtools/build/lib/skyframe/ActionLookupValuesTraversal.java index 7d61c7ccba37f5..046347399bd499 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/ActionLookupValuesTraversal.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ActionLookupValuesTraversal.java @@ -25,6 +25,7 @@ import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos; import com.google.devtools.build.lib.concurrent.Sharder; import com.google.devtools.build.skyframe.SkyValue; +import java.util.Objects; /** Represents the traversal of the ActionLookupValues in a build. */ public class ActionLookupValuesTraversal { @@ -43,7 +44,7 @@ public class ActionLookupValuesTraversal { public ActionLookupValuesTraversal() {} - void accumulate(ActionLookupKeyOrProxy keyForDebugging, SkyValue value) { + void accumulate(ActionLookupKeyOrProxy key, SkyValue value) { boolean isConfiguredTarget = value instanceof ConfiguredTargetValue; boolean isActionLookupValue = value instanceof ActionLookupValue; if (!isConfiguredTarget && !isActionLookupValue) { @@ -52,7 +53,16 @@ void accumulate(ActionLookupKeyOrProxy keyForDebugging, SkyValue value) { String.format( "Should only be called with ConfiguredTargetValue or ActionLookupValue: %s %s" + " %s", - value.getClass(), keyForDebugging, value))); + value.getClass(), key, value))); + return; + } + if (isConfiguredTarget + && !Objects.equals( + key.getConfigurationKey(), + ((ConfiguredTargetValue) value).getConfiguredTarget().getConfigurationKey())) { + // The configuration of the key doesn't match the configuration of the value. This means that + // the ConfiguredTargetValue is delegated from a different key. This ConfiguredTargetValue + // will show up again under its own key. Avoids double counting by skipping accumulation. return; } configuredObjectCount++; @@ -72,8 +82,7 @@ void accumulate(ActionLookupKeyOrProxy keyForDebugging, SkyValue value) { if (!(value instanceof NonRuleConfiguredTargetValue)) { BugReport.sendBugReport( new IllegalStateException( - String.format( - "Unexpected value type: %s %s %s", value.getClass(), keyForDebugging, value))); + String.format("Unexpected value type: %s %s %s", value.getClass(), key, value))); return; } ConfiguredTarget configuredTarget = diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BUILD b/src/main/java/com/google/devtools/build/lib/skyframe/BUILD index 0838022f2d0d18..c76d8274a798c9 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/BUILD +++ b/src/main/java/com/google/devtools/build/lib/skyframe/BUILD @@ -236,6 +236,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:config/build_configuration", "//src/main/java/com/google/devtools/build/lib/analysis:config/build_options", "//src/main/java/com/google/devtools/build/lib/analysis:config/config_conditions", + "//src/main/java/com/google/devtools/build/lib/analysis:config/configuration_value_event", "//src/main/java/com/google/devtools/build/lib/analysis:config/core_options", "//src/main/java/com/google/devtools/build/lib/analysis:config/execution_transition_factory", "//src/main/java/com/google/devtools/build/lib/analysis:config/fragment_factory", @@ -247,10 +248,8 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:config/toolchain_type_requirement", "//src/main/java/com/google/devtools/build/lib/analysis:config/transitions/baseline_options_value", "//src/main/java/com/google/devtools/build/lib/analysis:config/transitions/configuration_transition", - "//src/main/java/com/google/devtools/build/lib/analysis:config/transitions/null_transition", "//src/main/java/com/google/devtools/build/lib/analysis:config/transitions/patch_transition", "//src/main/java/com/google/devtools/build/lib/analysis:config/transitions/transition_collector", - "//src/main/java/com/google/devtools/build/lib/analysis:configurations_collector", "//src/main/java/com/google/devtools/build/lib/analysis:configured_object_value", "//src/main/java/com/google/devtools/build/lib/analysis:configured_target", "//src/main/java/com/google/devtools/build/lib/analysis:configured_target_value", diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BuildConfigurationFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/BuildConfigurationFunction.java index d90d21a95fc3d2..55a3f0fe134d94 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/BuildConfigurationFunction.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/BuildConfigurationFunction.java @@ -25,6 +25,7 @@ import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationValueEvent; import com.google.devtools.build.lib.analysis.config.CoreOptions; import com.google.devtools.build.lib.analysis.config.FragmentFactory; import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; @@ -105,15 +106,19 @@ public SkyValue compute(SkyKey skyKey, Environment env) } try { - return BuildConfigurationValue.create( - targetOptions, - RepositoryName.createUnvalidated(workspaceNameValue.getName()), - starlarkSemantics.getBool(BuildLanguageOptions.EXPERIMENTAL_SIBLING_REPOSITORY_LAYOUT), - transitionDirectoryNameFragment, - // Arguments below this are server-global. - directories, - ruleClassProvider, - fragmentFactory); + var configurationValue = + BuildConfigurationValue.create( + targetOptions, + RepositoryName.createUnvalidated(workspaceNameValue.getName()), + starlarkSemantics.getBool( + BuildLanguageOptions.EXPERIMENTAL_SIBLING_REPOSITORY_LAYOUT), + transitionDirectoryNameFragment, + // Arguments below this are server-global. + directories, + ruleClassProvider, + fragmentFactory); + env.getListener().post(ConfigurationValueEvent.create(configurationValue)); + return configurationValue; } catch (InvalidConfigurationException e) { throw new BuildConfigurationFunctionException(e); } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java index 9b9756ce198f20..3cf6d615c8c17f 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java @@ -284,11 +284,14 @@ public SkyValue compute(SkyKey key, Environment env) // If this CT applied an incoming rule transition, log it. BuildConfigurationValue config = prereqs.getTargetAndConfiguration().getConfiguration(); - if (config != null && !config.getKey().equals(configuredTargetKey.getConfigurationKey())) { + // `toKey` here retrieves the original key if there's a proxy key due to a transition. + if (config != null + && !config.getKey().equals(configuredTargetKey.toKey().getConfigurationKey())) { env.getListener() .post( new ConfigRequestedEvent( - config, configuredTargetKey.getConfigurationKey().getOptionsChecksum())); + config, + configuredTargetKey.toKey().getConfigurationKey().getOptionsChecksum())); } // If one of our dependencies is platform-incompatible with this build, so are we. diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java index bdf21b7fa4ba42..a78cd7a9db7160 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java @@ -511,7 +511,11 @@ public void dumpSkyframeStateInParallel( if (skyValue instanceof RuleConfiguredTargetValue) { tasks.add( () -> { - actionGraphDump.dumpConfiguredTarget((RuleConfiguredTargetValue) skyValue); + var configuredTarget = (RuleConfiguredTargetValue) skyValue; + // Only dumps the value for non-delegating keys. + if (configuredTarget.getConfiguredTarget().getKeyOrProxy().equals(key)) { + actionGraphDump.dumpConfiguredTarget(configuredTarget); + } return null; }); } else if (key.functionName().equals(SkyFunctions.ASPECT)) { @@ -570,7 +574,11 @@ public void dumpSkyframeState(ActionGraphDump actionGraphDump) } try { if (skyValue instanceof RuleConfiguredTargetValue) { - actionGraphDump.dumpConfiguredTarget((RuleConfiguredTargetValue) skyValue); + var configuredTarget = (RuleConfiguredTargetValue) skyValue; + // Only dumps the value for non-delegating keys. + if (configuredTarget.getConfiguredTarget().getKeyOrProxy().equals(key)) { + actionGraphDump.dumpConfiguredTarget(configuredTarget); + } } else if (key.functionName().equals(SkyFunctions.ASPECT)) { AspectValue aspectValue = (AspectValue) skyValue; AspectKey aspectKey = (AspectKey) key; diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java index 5a949309c04f1e..018f80ee686805 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java @@ -78,10 +78,7 @@ import com.google.devtools.build.lib.analysis.AnalysisOptions; import com.google.devtools.build.lib.analysis.AspectConfiguredEvent; import com.google.devtools.build.lib.analysis.AspectValue; -import com.google.devtools.build.lib.analysis.BaseDependencySpecification; import com.google.devtools.build.lib.analysis.BlazeDirectories; -import com.google.devtools.build.lib.analysis.ConfigurationsCollector; -import com.google.devtools.build.lib.analysis.ConfigurationsResult; import com.google.devtools.build.lib.analysis.ConfiguredAspect; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.ConfiguredTarget; @@ -99,20 +96,14 @@ import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory; import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; import com.google.devtools.build.lib.analysis.config.BuildOptions; -import com.google.devtools.build.lib.analysis.config.ConfigurationResolver; import com.google.devtools.build.lib.analysis.config.CoreOptions; import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; -import com.google.devtools.build.lib.analysis.config.transitions.ConfigurationTransition; -import com.google.devtools.build.lib.analysis.config.transitions.NullTransition; import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget; import com.google.devtools.build.lib.analysis.constraints.RuleContextConstraintSemantics; import com.google.devtools.build.lib.analysis.producers.ConfiguredTargetAndDataProducer; import com.google.devtools.build.lib.analysis.producers.DependencyError; import com.google.devtools.build.lib.analysis.producers.DependencyMapProducer; import com.google.devtools.build.lib.analysis.producers.PrerequisiteParameters; -import com.google.devtools.build.lib.analysis.starlark.StarlarkBuildSettingsDetailsValue; -import com.google.devtools.build.lib.analysis.starlark.StarlarkTransition; -import com.google.devtools.build.lib.analysis.starlark.StarlarkTransition.TransitionException; import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleValue; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; import com.google.devtools.build.lib.bugreport.BugReport; @@ -132,7 +123,6 @@ import com.google.devtools.build.lib.concurrent.ThreadSafety; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; import com.google.devtools.build.lib.events.ErrorSensingEventHandler; -import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.devtools.build.lib.events.Reporter; @@ -293,7 +283,7 @@ * additional artifacts (workspace status and build info artifacts) into SkyFunctions for use during * the build. */ -public abstract class SkyframeExecutor implements WalkableGraphFactory, ConfigurationsCollector { +public abstract class SkyframeExecutor implements WalkableGraphFactory { private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); protected MemoizingEvaluator memoizingEvaluator; @@ -1806,126 +1796,6 @@ public Map getConfigurations( Functions.identity(), key -> (BuildConfigurationValue) evaluationResult.get(key))); } - /** - * Retrieves the configurations needed for the given deps. Unconditionally includes all fragments. - * - *

Skips targets with loading phase errors. - */ - // TODO(ulfjack): Remove this legacy method after switching to the Skyframe-based implementation. - @Override - public ConfigurationsResult getConfigurations( - ExtendedEventHandler eventHandler, - BuildOptions fromOptions, - Iterable keys) - throws InvalidConfigurationException, InterruptedException { - ConfigurationsResult.Builder builder = ConfigurationsResult.newBuilder(); - - PlatformMappingValue platformMappingValue = getPlatformMappingValue(eventHandler, fromOptions); - - // Now get the configurations. - List configSkyKeys = new ArrayList<>(); - for (BaseDependencySpecification key : keys) { - ConfigurationTransition transition = key.getTransition(); - if (transition == NullTransition.INSTANCE) { - continue; - } - Collection toOptions; - try { - StarlarkBuildSettingsDetailsValue details = - getStarlarkBuildSettingsDetailsValue(eventHandler, transition); - toOptions = - ConfigurationResolver.applyTransitionWithoutSkyframe( - fromOptions, - transition, - details, - eventHandler, - skyframeBuildView.getStarlarkTransitionCache()) - .values(); - } catch (TransitionException e) { - eventHandler.handle(Event.error(e.getMessage())); - builder.setHasError(); - continue; - } - for (BuildOptions toOption : toOptions) { - configSkyKeys.add(toConfigurationKey(platformMappingValue, toOption)); - } - } - - EvaluationResult configsResult = - evaluateSkyKeys(eventHandler, configSkyKeys, /*keepGoing=*/ true); - - for (BaseDependencySpecification key : keys) { - if (key.getTransition() == NullTransition.INSTANCE) { - builder.put(key, null); - continue; - } - Collection toOptions; - try { - StarlarkBuildSettingsDetailsValue details = - getStarlarkBuildSettingsDetailsValue(eventHandler, key.getTransition()); - toOptions = - ConfigurationResolver.applyTransitionWithoutSkyframe( - fromOptions, - key.getTransition(), - details, - eventHandler, - skyframeBuildView.getStarlarkTransitionCache()) - .values(); - } catch (TransitionException e) { - eventHandler.handle(Event.error(e.getMessage())); - builder.setHasError(); - continue; - } - - for (BuildOptions toOption : toOptions) { - BuildConfigurationKey configKey = toConfigurationKey(platformMappingValue, toOption); - BuildConfigurationValue configValue = - (BuildConfigurationValue) configsResult.get(configKey); - if (configValue != null) { - builder.put(key, configValue); - } else if (configsResult.errorMap().containsKey(configKey)) { - ErrorInfo configError = configsResult.getError(configKey); - if (configError.getException() instanceof InvalidConfigurationException) { - // Wrap underlying exception to make it clearer to developers which line of code - // actually threw exception. - InvalidConfigurationException underlying = - (InvalidConfigurationException) configError.getException(); - throw new InvalidConfigurationException(underlying.getDetailedExitCode(), underlying); - } - } - } - } - return builder.build(); - } - - /** Must be in sync with {@link ConfigurationResolver#getStarlarkBuildSettingsDetailsValue}. */ - private StarlarkBuildSettingsDetailsValue getStarlarkBuildSettingsDetailsValue( - ExtendedEventHandler eventHandler, ConfigurationTransition transition) - throws TransitionException { - ImmutableSet

Unconditionally includes all fragments. - */ - public BuildConfigurationValue getConfigurationForTesting( - Target target, BuildConfigurationValue config, ExtendedEventHandler eventHandler) - throws InvalidConfigurationException, InterruptedException { - List node = - ImmutableList.of(new TargetAndConfiguration(target, config)); - Collection configs = - ConfigurationResolver.getConfigurationsFromExecutor( - node, - AnalysisUtils.targetsToDeps(new LinkedHashSet<>(node), ruleClassProvider), - eventHandler, - skyframeExecutor) - .getTargetsAndConfigs(); - return configs.iterator().next().getConfiguration(); - } - /** * Sets the possible artifact roots in the artifact factory. This allows the factory to resolve * paths with unknown roots to artifacts. diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java index 4a99b4399886f7..c5866037394013 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java @@ -2006,17 +2006,11 @@ protected BuildConfigurationValue getConfiguration( } private BuildConfigurationValue getConfiguration(String label) { - BuildConfigurationValue config; try { - config = getConfiguration(getConfiguredTarget(label)); - config = view.getConfigurationForTesting(getTarget(label), config, reporter); + return getConfiguration(getConfiguredTarget(label)); } catch (LabelSyntaxException e) { throw new IllegalArgumentException(e); - } catch (Exception e) { - // TODO(b/36585204): Clean this up - throw new RuntimeException(e); } - return config; } protected final BuildConfigurationValue getConfiguration(BuildConfigurationKey configurationKey) { diff --git a/src/test/shell/integration/starlark_configurations_test.sh b/src/test/shell/integration/starlark_configurations_test.sh index 37a8addd508a9d..b1a6c3147903ae 100755 --- a/src/test/shell/integration/starlark_configurations_test.sh +++ b/src/test/shell/integration/starlark_configurations_test.sh @@ -621,7 +621,7 @@ load('//test:rule.bzl', 'my_rule') my_rule(name = 'test') EOF bazel build //test:test >& "$TEST_log" || exit_code="$?" - assert_equals 2 "$exit_code" || fail "Expected exit code 2" + assert_equals 1 "$exit_code" || fail "Expected exit code 1" expect_log "CPU name '//bad:cpu'" expect_log "is invalid as part of a path: must not contain /" } From 94c03155d6c427ec0e2328eb0272dcb6470f394d Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 09:05:13 -0700 Subject: [PATCH 36/51] Add `SkyValueDirtinessChecker` utility which returns the mtsv for `DirectoryFileStateValue` nodes PiperOrigin-RevId: 545692515 Change-Id: Ia566b68d6551587cdc2c0619bf9552bc6fbb62d2 --- .../skyframe/FileSystemValueCheckerInferringAncestors.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestors.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestors.java index 14f3571ff33788..eba5c66214fd5e 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestors.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestors.java @@ -279,8 +279,11 @@ && listingHasEntriesOutsideOf(path, maybeDeletedChildren))) { if (oldFsv.getType().isDirectory()) { return false; } - // TODO(b/287632270) - handle this scenario - valuesToInject.put(key, Delta.justNew(FileStateValue.DIRECTORY_FILE_STATE_NODE)); + Version directoryFileStateNodeMtsv = + skyValueDirtinessChecker.getMaxTransitiveSourceVersionForNewValue( + key, FileStateValue.DIRECTORY_FILE_STATE_NODE); + valuesToInject.put( + key, Delta.justNew(FileStateValue.DIRECTORY_FILE_STATE_NODE, directoryFileStateNodeMtsv)); parentListingKey(path).ifPresent(valuesToInvalidate::add); return true; } From 7119b390e10ce5939bd9005972997aad97041922 Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 09:24:47 -0700 Subject: [PATCH 37/51] Use `paths.bzl` instead of introducing a new method for replacing file extensions PiperOrigin-RevId: 545697933 Change-Id: I3afd3b8fa80ac888265a49b0cd63a5cfdad82f09 --- .../starlark/builtins_bzl/common/java/java_common.bzl | 4 ++-- .../common/java/java_common_internal_for_builtins.bzl | 4 ++-- .../starlark/builtins_bzl/common/java/java_helper.bzl | 8 -------- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/main/starlark/builtins_bzl/common/java/java_common.bzl b/src/main/starlark/builtins_bzl/common/java/java_common.bzl index 91490fff8c68e5..3d0f32065c1a3c 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_common.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_common.bzl @@ -18,7 +18,7 @@ load(":common/java/java_info.bzl", "JavaInfo") load(":common/java/java_common_internal_for_builtins.bzl", "compile", "merge", "run_ijar") load(":common/java/java_plugin_info.bzl", "JavaPluginInfo") load(":common/java/java_semantics.bzl", "semantics") -load(":common/java/java_helper.bzl", "helper") +load(":common/paths.bzl", "paths") _java_common_internal = _builtins.internal.java_common_internal_do_not_use @@ -97,7 +97,7 @@ def _stamp_jar(actions, jar, java_toolchain, target_label): (File) The output artifact """ - output = actions.declare_file(helper.basename_without_extension(jar) + "-stamped.jar", sibling = jar) + output = actions.declare_file(paths.replace_extension(jar.basename, "-stamped.jar"), sibling = jar) args = actions.args() args.add(jar) args.add(output) diff --git a/src/main/starlark/builtins_bzl/common/java/java_common_internal_for_builtins.bzl b/src/main/starlark/builtins_bzl/common/java/java_common_internal_for_builtins.bzl index 768e4ae97f4af4..663c103571b35a 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_common_internal_for_builtins.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_common_internal_for_builtins.bzl @@ -15,7 +15,7 @@ """ Private utilities for Java compilation support in Starlark. """ load(":common/java/java_semantics.bzl", "semantics") -load(":common/java/java_helper.bzl", "helper") +load(":common/paths.bzl", "paths") _java_common_internal = _builtins.internal.java_common_internal_do_not_use @@ -158,7 +158,7 @@ def run_ijar( (File) The output artifact """ if not output: - output = actions.declare_file(helper.basename_without_extension(jar) + "-ijar.jar", sibling = jar) + output = actions.declare_file(paths.replace_extension(jar.basename, "-ijar.jar"), sibling = jar) args = actions.args() args.add(jar) args.add(output) diff --git a/src/main/starlark/builtins_bzl/common/java/java_helper.bzl b/src/main/starlark/builtins_bzl/common/java/java_helper.bzl index cee33cd68f02bc..eebc50d57dec67 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_helper.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_helper.bzl @@ -330,13 +330,6 @@ def _create_single_jar(ctx, output, *input_depsets): def _shell_quote(s): return "'" + s.replace("'", "'\\''") + "'" -def _basename_without_extension(file): - before, sep, after = file.basename.rpartition(".") - if sep: - return before - else: - return after - helper = struct( collect_all_targets_as_deps = _collect_all_targets_as_deps, filter_launcher_for_target = _filter_launcher_for_target, @@ -359,5 +352,4 @@ helper = struct( test_providers = _test_providers, create_single_jar = _create_single_jar, shell_quote = _shell_quote, - basename_without_extension = _basename_without_extension, ) From b3e01062842218447af2024eef4d098bde1620bd Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 09:46:46 -0700 Subject: [PATCH 38/51] Refactor test helpers in `FileSystemValueCheckerInferringAncestorsTest` and `FileSystemValueCheckerInferringAncestorsTestBase` PiperOrigin-RevId: 545704347 Change-Id: I5a98023122cda628d424a48b46383e3663470d7b --- ...temValueCheckerInferringAncestorsTest.java | 30 ++++++++++- ...alueCheckerInferringAncestorsTestBase.java | 52 ++++--------------- 2 files changed, 38 insertions(+), 44 deletions(-) diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestorsTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestorsTest.java index 138dfa8acba290..84361b27fcd31b 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestorsTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestorsTest.java @@ -23,6 +23,7 @@ import static com.google.devtools.build.lib.testing.common.DirectoryListingHelper.symlink; import static org.junit.Assert.assertThrows; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; @@ -36,11 +37,18 @@ import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.skyframe.Differencer.DiffWithDelta.Delta; import com.google.devtools.build.skyframe.ImmutableDiff; +import com.google.devtools.build.skyframe.InMemoryNodeEntry; +import com.google.devtools.build.skyframe.NodeBatch; +import com.google.devtools.build.skyframe.NodeEntry.DirtyType; +import com.google.devtools.build.skyframe.QueryableGraph.Reason; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; +import com.google.devtools.build.skyframe.Version; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.testing.junit.testparameterinjector.TestParameter; import com.google.testing.junit.testparameterinjector.TestParameterInjector; import java.io.IOException; +import java.util.Map.Entry; import org.junit.Test; import org.junit.runner.RunWith; @@ -720,7 +728,27 @@ private Delta fileStateValueDelta(String relativePath) throws IOException { return Delta.justNew(fileStateValue(relativePath)); } + private void addDoneNodesAndThenMarkChanged(ImmutableMap values) + throws InterruptedException { + for (Entry entry : values.entrySet()) { + InMemoryNodeEntry node = addDoneNode(entry.getKey(), entry.getValue()); + node.markDirty(DirtyType.CHANGE); + } + } + private void addDoneNodes(ImmutableMap values) throws InterruptedException { - addDoneNodes(values, /* mtsv= */ null); + for (Entry entry : values.entrySet()) { + addDoneNode(entry.getKey(), entry.getValue()); + } + } + + @CanIgnoreReturnValue + private InMemoryNodeEntry addDoneNode(SkyKey key, SkyValue value) throws InterruptedException { + NodeBatch batch = inMemoryGraph.createIfAbsentBatch(null, Reason.OTHER, ImmutableList.of(key)); + InMemoryNodeEntry entry = (InMemoryNodeEntry) batch.get(key); + entry.addReverseDepAndCheckIfDone(null); + entry.markRebuilding(); + entry.setValue(value, Version.minimal(), /* maxTransitiveSourceVersion= */ null); + return entry; } } diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestorsTestBase.java b/src/test/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestorsTestBase.java index aa871af188344f..8369676bc77c21 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestorsTestBase.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestorsTestBase.java @@ -18,7 +18,6 @@ import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.actions.FileStateValue; import com.google.devtools.build.lib.testutil.Scratch; import com.google.devtools.build.lib.vfs.DelegateFileSystem; @@ -32,18 +31,10 @@ import com.google.devtools.build.lib.vfs.RootedPath; import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.skyframe.InMemoryGraph; -import com.google.devtools.build.skyframe.InMemoryNodeEntry; -import com.google.devtools.build.skyframe.NodeBatch; -import com.google.devtools.build.skyframe.NodeEntry.DirtyType; -import com.google.devtools.build.skyframe.QueryableGraph.Reason; -import com.google.devtools.build.skyframe.SkyKey; -import com.google.devtools.build.skyframe.SkyValue; -import com.google.devtools.build.skyframe.Version; -import com.google.errorprone.annotations.CanIgnoreReturnValue; +import com.google.errorprone.annotations.ForOverride; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Map.Entry; import javax.annotation.Nullable; import org.junit.After; import org.junit.Before; @@ -53,13 +44,14 @@ public class FileSystemValueCheckerInferringAncestorsTestBase { protected final List statedPaths = new ArrayList<>(); protected DefaultSyscallCache syscallCache = DefaultSyscallCache.newBuilder().build(); protected Root root; - protected final InMemoryGraph inMemoryGraph = InMemoryGraph.create(); + protected InMemoryGraph inMemoryGraph; private Root untrackedRoot; Exception throwOnStat; @Before - public void createRoot() throws IOException { + public void setUpGraphAndRoot() throws IOException { + createGraph(); Path srcRootPath = scratch.dir("/src"); PathFragment srcRoot = srcRootPath.asFragment(); FileSystem trackingFileSystem = @@ -83,6 +75,11 @@ public synchronized FileStatus statIfFound(PathFragment path, boolean followSyml untrackedRoot = Root.fromPath(srcRootPath); } + @ForOverride + protected void createGraph() { + inMemoryGraph = InMemoryGraph.create(); + } + @After public void checkExceptionThrown() { assertThat(throwOnStat).isNull(); @@ -110,35 +107,4 @@ protected FileStateValue fileStateValue(String relativePath) throws IOException protected static DirectoryListingStateValue directoryListingStateValue(Dirent... dirents) { return DirectoryListingStateValue.create(ImmutableList.copyOf(dirents)); } - - protected void addDoneNodesAndThenMarkChanged(ImmutableMap values) - throws InterruptedException { - addDoneNodesAndThenMarkChanged(values, /* mtsv= */ null); - } - - protected void addDoneNodesAndThenMarkChanged( - ImmutableMap values, @Nullable Version mtsv) throws InterruptedException { - for (Entry entry : values.entrySet()) { - InMemoryNodeEntry node = addDoneNode(entry.getKey(), entry.getValue(), mtsv); - node.markDirty(DirtyType.CHANGE); - } - } - - protected void addDoneNodes(ImmutableMap values, @Nullable Version mtsv) - throws InterruptedException { - for (Entry entry : values.entrySet()) { - addDoneNode(entry.getKey(), entry.getValue(), mtsv); - } - } - - @CanIgnoreReturnValue - private InMemoryNodeEntry addDoneNode(SkyKey key, SkyValue value, @Nullable Version mtsv) - throws InterruptedException { - NodeBatch batch = inMemoryGraph.createIfAbsentBatch(null, Reason.OTHER, ImmutableList.of(key)); - InMemoryNodeEntry entry = (InMemoryNodeEntry) batch.get(key); - entry.addReverseDepAndCheckIfDone(null); - entry.markRebuilding(); - entry.setValue(value, Version.minimal(), /* maxTransitiveSourceVersion= */ mtsv); - return entry; - } } From f2e9c78a919d18f18a07cf253258f84f5a8d4052 Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 10:13:08 -0700 Subject: [PATCH 39/51] Implement `java_common.add_constraints` in Starlark and delete corresponding native code PiperOrigin-RevId: 545712006 Change-Id: I019dad243b1fc2e939a046eda5f8d40cd401649a --- .../lib/rules/java/JavaStarlarkCommon.java | 8 ------- .../starlarkbuildapi/java/JavaCommonApi.java | 21 ---------------- .../builtins_bzl/common/java/java_common.bzl | 11 +++++++-- .../builtins_bzl/common/java/java_info.bzl | 24 +++++++++++++++++++ 4 files changed, 33 insertions(+), 31 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java index 4266567efaa752..2573dd9152b738 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java @@ -283,14 +283,6 @@ public ProviderApi getMessageBundleInfo() { return null; } - @Override - public Info addConstraints(Info info, Sequence constraints) - throws EvalException, RuleErrorException { - // No implementation in Bazel. This method is not callable in Starlark except through - // (discouraged) use of --experimental_google_legacy_api. - return info; - } - @Override public Sequence getConstraints(Info info) throws RuleErrorException { // No implementation in Bazel. This method not callable in Starlark except through diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java index 700f54e0467800..8be39f1eed7e6d 100644 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java @@ -471,27 +471,6 @@ JavaInfoT mergeJavaProviders( enableOnlyWithFlag = BuildLanguageOptions.EXPERIMENTAL_GOOGLE_LEGACY_API) ProviderApi getMessageBundleInfo(); - @StarlarkMethod( - name = "add_constraints", - doc = "Returns a copy of the given JavaInfo with the given constraints added.", - parameters = { - @Param( - name = "java_info", - positional = true, - named = false, - doc = "The JavaInfo to enhance."), - @Param( - name = "constraints", - allowedTypes = {@ParamType(type = Sequence.class, generic1 = String.class)}, - named = true, - positional = false, - defaultValue = "[]", - doc = "Constraints to add") - }, - enableOnlyWithFlag = BuildLanguageOptions.EXPERIMENTAL_GOOGLE_LEGACY_API) - Info addConstraints(Info javaInfo, Sequence constraints /* expected. */) - throws EvalException, RuleErrorException; - @StarlarkMethod( name = "get_constraints", doc = "Returns a set of constraints added.", diff --git a/src/main/starlark/builtins_bzl/common/java/java_common.bzl b/src/main/starlark/builtins_bzl/common/java/java_common.bzl index 3d0f32065c1a3c..3a7bada71da402 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_common.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_common.bzl @@ -14,7 +14,11 @@ """ Utilities for Java compilation support in Starlark. """ -load(":common/java/java_info.bzl", "JavaInfo") +load( + ":common/java/java_info.bzl", + "JavaInfo", + _java_info_add_constraints = "add_constraints", +) load(":common/java/java_common_internal_for_builtins.bzl", "compile", "merge", "run_ijar") load(":common/java/java_plugin_info.bzl", "JavaPluginInfo") load(":common/java/java_semantics.bzl", "semantics") @@ -182,7 +186,10 @@ def _add_constraints(java_info, constraints = []): Returns: (JavaInfo) """ - return _java_common_internal.add_constraints(java_info, constraints = constraints) + if semantics.IS_BAZEL: + return java_info + + return _java_info_add_constraints(java_info, constraints = constraints) def _get_constraints(java_info): """Returns a set of constraints added. diff --git a/src/main/starlark/builtins_bzl/common/java/java_info.bzl b/src/main/starlark/builtins_bzl/common/java/java_info.bzl index 078117ab83a4a2..39cf7031476596 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_info.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_info.bzl @@ -146,6 +146,30 @@ def to_java_binary_info(java_info): ) return _new_javainfo(**result) +def _to_mutable_dict(java_info): + return { + key: getattr(java_info, key) + for key in dir(java_info) + if key not in ["to_json", "to_proto"] + } + +def add_constraints(java_info, constraints = []): + """Returns a copy of the given JavaInfo with the given constraints added. + + Args: + java_info: (JavaInfo) The JavaInfo to enhance + constraints: ([str]) Constraints to add + + Returns: + (JavaInfo) + """ + result = _to_mutable_dict(java_info) + old_constraints = java_info._constraints if java_info._constraints else [] + result.update( + _constraints = depset(constraints + old_constraints).to_list(), + ) + return _new_javainfo(**result) + def _validate_provider_list(provider_list, what, expected_provider_type): _java_common_internal.check_provider_instances(provider_list, what, expected_provider_type) From 91d884ed5c6491931d5a82001cc1e925f39aa2aa Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 10:16:55 -0700 Subject: [PATCH 40/51] Delete `AndroidLintActionBuilder.java` Now that `java_binary` is only in Starlark, this class is no longer used PiperOrigin-RevId: 545713148 Change-Id: I72ae250acbec569659b9c62a7da5a572c8c5c9ea --- .../rules/java/AndroidLintActionBuilder.java | 153 ------------------ .../devtools/build/lib/rules/java/BUILD | 2 - 2 files changed, 155 deletions(-) delete mode 100644 src/main/java/com/google/devtools/build/lib/rules/java/AndroidLintActionBuilder.java diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/AndroidLintActionBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/java/AndroidLintActionBuilder.java deleted file mode 100644 index abc19aed79a220..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/rules/java/AndroidLintActionBuilder.java +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2020 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.lib.rules.java; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static java.nio.charset.StandardCharsets.UTF_8; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.actions.ExecutionRequirements; -import com.google.devtools.build.lib.actions.ParamFileInfo; -import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; -import com.google.devtools.build.lib.analysis.RuleContext; -import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; -import com.google.devtools.build.lib.analysis.actions.SpawnAction; -import com.google.devtools.build.lib.collect.nestedset.NestedSet; -import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; -import com.google.devtools.build.lib.packages.TargetUtils; -import com.google.devtools.build.lib.util.StringCanonicalizer; -import javax.annotation.Nullable; - -/** Helper to create Android Lint actions. */ -class AndroidLintActionBuilder { - private AndroidLintActionBuilder() {} - - private static final String MNEMONIC = "AndroidLint"; - private static final ParamFileInfo PARAM_FILE_INFO = - ParamFileInfo.builder(ParameterFileType.UNQUOTED) - .setCharset(UTF_8) - .setUseAlways(true) // needed to support workers - .build(); - - /** Creates and registers Android Lint action if needed and returns action's output if created. */ - @Nullable - static Artifact create( - RuleContext ruleContext, - JavaConfiguration config, - JavaTargetAttributes attributes, - BootClassPathInfo bootClassPathInfo, - JavaCommon common, - JavaSemantics semantics, - JavaCompileOutputs outputs) { - if (!config.runAndroidLint() - || !attributes.hasSources() - || JavaCommon.isNeverLink(ruleContext)) { - // Note Javac doesn't run when depending on neverlink library, so we also skip Android Lint. - return null; - } - if (config.limitAndroidLintToAndroidCompatible() - && !JavaCommon.getConstraints(ruleContext).contains("android")) { - return null; - } - JavaToolchainProvider toolchain = JavaToolchainProvider.from(ruleContext); - AndroidLintTool androidLint = toolchain.getAndroidLint(); - if (androidLint == null) { - ruleContext.ruleError( - "android_lint_wrapper not set in java_toolchain: " + toolchain.getToolchainLabel()); - return null; - } - - ImmutableList allSrcJars = attributes.getSourceJars(); - if (outputs.genSource() != null) { - allSrcJars = - ImmutableList.builder().addAll(allSrcJars).add(outputs.genSource()).build(); - } - NestedSet classpath = attributes.getCompileTimeClassPath(); - if (!bootClassPathInfo.auxiliary().isEmpty()) { - classpath = - NestedSetBuilder.naiveLinkOrder() - .addTransitive(bootClassPathInfo.auxiliary()) - .addTransitive(classpath) - .build(); - } - - CustomCommandLine.Builder cmd = CustomCommandLine.builder(); - cmd.addExecPaths("--sources", attributes.getSourceFiles()) - .addExecPaths("--source_jars", allSrcJars) - .addExecPaths("--bootclasspath", bootClassPathInfo.bootclasspath()) - .addExecPaths("--classpath", classpath) - .addExecPaths("--plugins", attributes.plugins().plugins().processorClasspath()) - .addLabel("--target_label", ruleContext.getLabel()); - ImmutableList javacopts = - common.getJavacOpts().stream().map(StringCanonicalizer::intern).collect(toImmutableList()); - if (!javacopts.isEmpty()) { - cmd.addAll("--javacopts", javacopts); - // terminate --javacopts with `--` to support javac flags that start with `--` - cmd.add("--"); - } - cmd.add("--lintopts"); - cmd.addAll(androidLint.options()); - - SpawnAction.Builder spawnAction = new SpawnAction.Builder(); - for (JavaPackageConfigurationProvider provider : androidLint.packageConfiguration()) { - if (provider.matches(ruleContext.getLabel())) { - cmd.addAll(provider.javacopts()); - spawnAction.addTransitiveInputs(provider.data()); - } - } - - Artifact result = - ruleContext.getPackageRelativeArtifact( - ruleContext.getLabel().getName() + "_android_lint_output.xml", - ruleContext.getBinOrGenfilesDirectory()); - cmd.addExecPath("--xml", result); - - NestedSetBuilder toolInputs = NestedSetBuilder.stableOrder(); - androidLint.tool().addInputs(toolchain, toolInputs); - - semantics.setLintProgressMessage(spawnAction); - ruleContext.registerAction( - spawnAction - .addCommandLine( - androidLint.tool().getCommandLine(toolchain, /* stripOutputPath= */ null)) - .addCommandLine(cmd.build(), PARAM_FILE_INFO) - .addInputs(attributes.getSourceFiles()) - .addInputs(allSrcJars) - .addTransitiveInputs(bootClassPathInfo.bootclasspath()) - .addTransitiveInputs(classpath) - .addTransitiveInputs(attributes.plugins().plugins().processorClasspath()) - .addTransitiveInputs(attributes.plugins().plugins().data()) - .addTransitiveTools(toolInputs.build()) - .addOutput(result) - .setMnemonic(MNEMONIC) - .setExecutionInfo(getExecutionInfo(ruleContext)) - .build(ruleContext)); - return result; - } - - /** Advertises worker support added in b/191156225. */ - private static ImmutableMap getExecutionInfo(RuleContext ruleContext) { - ImmutableMap executionInfo = - ImmutableMap.of(ExecutionRequirements.SUPPORTS_WORKERS, "1"); - - return ImmutableMap.builder() - .putAll(ruleContext.getConfiguration().modifiedExecutionInfo(executionInfo, MNEMONIC)) - .putAll( - TargetUtils.getExecutionInfo( - ruleContext.getRule(), ruleContext.isAllowTagsPropagation())) - .buildOrThrow(); - } -} diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/BUILD b/src/main/java/com/google/devtools/build/lib/rules/java/BUILD index 98f41cbc2fbf9e..3e4a4771613c54 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/BUILD +++ b/src/main/java/com/google/devtools/build/lib/rules/java/BUILD @@ -73,7 +73,6 @@ java_library( java_library( name = "java-compilation", srcs = [ - "AndroidLintActionBuilder.java", "AndroidLintTool.java", "BootClassPathInfo.java", "BuildInfoPropertiesTranslator.java", @@ -175,7 +174,6 @@ java_library( "//src/main/java/com/google/devtools/build/lib/util", "//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code", "//src/main/java/com/google/devtools/build/lib/util:filetype", - "//src/main/java/com/google/devtools/build/lib/util:string", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//src/main/java/com/google/devtools/common/options", From 691b48b86da029286a4f1dc42963a8c49203ad27 Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 10:42:54 -0700 Subject: [PATCH 41/51] Implement `java_common.get_constraints` in Starlark and delete corresponding native code PiperOrigin-RevId: 545721573 Change-Id: I109df5998ee641d2d85854ea0183e4c000df8773 --- .../build/lib/rules/java/JavaStarlarkCommon.java | 7 ------- .../lib/starlarkbuildapi/java/JavaCommonApi.java | 13 ------------- .../builtins_bzl/common/java/java_common.bzl | 2 +- 3 files changed, 1 insertion(+), 21 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java index 2573dd9152b738..6f1997e40803a4 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java @@ -283,13 +283,6 @@ public ProviderApi getMessageBundleInfo() { return null; } - @Override - public Sequence getConstraints(Info info) throws RuleErrorException { - // No implementation in Bazel. This method not callable in Starlark except through - // (discouraged) use of --experimental_google_legacy_api. - return StarlarkList.empty(); - } - @Override public JavaInfo setAnnotationProcessing( Info info, diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java index 8be39f1eed7e6d..d5a6f7cc3a83d2 100644 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java @@ -471,19 +471,6 @@ JavaInfoT mergeJavaProviders( enableOnlyWithFlag = BuildLanguageOptions.EXPERIMENTAL_GOOGLE_LEGACY_API) ProviderApi getMessageBundleInfo(); - @StarlarkMethod( - name = "get_constraints", - doc = "Returns a set of constraints added.", - parameters = { - @Param( - name = "java_info", - positional = true, - named = false, - doc = "The JavaInfo to get constraints from."), - }, - enableOnlyWithFlag = BuildLanguageOptions.EXPERIMENTAL_GOOGLE_LEGACY_API) - Sequence getConstraints(Info javaInfo) throws RuleErrorException; - @StarlarkMethod( name = "set_annotation_processing", doc = "Returns a copy of the given JavaInfo with the given annotation_processing info.", diff --git a/src/main/starlark/builtins_bzl/common/java/java_common.bzl b/src/main/starlark/builtins_bzl/common/java/java_common.bzl index 3a7bada71da402..5a8a89f338c736 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_common.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_common.bzl @@ -200,7 +200,7 @@ def _get_constraints(java_info): Returns: ([str]) The constraints set on the supplied JavaInfo """ - return _java_common_internal.get_constraints(java_info) + return [] if semantics.IS_BAZEL else java_info._constraints def _set_annotation_processing( java_info, From df82dcd1b57a9a7460156ff2d53b376737115dbc Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 10:56:46 -0700 Subject: [PATCH 42/51] Add the target path of an UnresolvedSymlink action to the aquery results Android converts bazel actions to a ninja build graph by querying the actions with aquery, but was unable to convert UnresolvedSymlink actions because the target path wasn't included in the aquery output. RELNOTES: Added the target path of an UnresolvedSymlink action to the aquery results PiperOrigin-RevId: 545726666 Change-Id: Ie106e5f5800abd035469f3572a8504e86dc7d3b4 --- .../starlark/UnresolvedSymlinkAction.java | 4 ++ ...ctionGraphTextOutputFormatterCallback.java | 8 ++++ .../actiongraph/v2/ActionGraphDump.java | 6 +++ src/main/protobuf/analysis_v2.proto | 4 ++ src/test/shell/integration/aquery_test.sh | 40 +++++++++++++++++++ 5 files changed, 62 insertions(+) diff --git a/src/main/java/com/google/devtools/build/lib/analysis/starlark/UnresolvedSymlinkAction.java b/src/main/java/com/google/devtools/build/lib/analysis/starlark/UnresolvedSymlinkAction.java index 95910b3413c29d..11a54bfad32735 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/starlark/UnresolvedSymlinkAction.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/starlark/UnresolvedSymlinkAction.java @@ -103,6 +103,10 @@ protected String getRawProgressMessage() { return progressMessage; } + public PathFragment getTarget() { + return target; + } + private static DetailedExitCode createDetailedExitCode(String message, Code detailedCode) { return DetailedExitCode.of( FailureDetail.newBuilder() diff --git a/src/main/java/com/google/devtools/build/lib/query2/aquery/ActionGraphTextOutputFormatterCallback.java b/src/main/java/com/google/devtools/build/lib/query2/aquery/ActionGraphTextOutputFormatterCallback.java index cb7763a5a3cb4f..e87ad3fec7f35a 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/aquery/ActionGraphTextOutputFormatterCallback.java +++ b/src/main/java/com/google/devtools/build/lib/query2/aquery/ActionGraphTextOutputFormatterCallback.java @@ -35,6 +35,7 @@ import com.google.devtools.build.lib.analysis.actions.ParameterFileWriteAction; import com.google.devtools.build.lib.analysis.actions.Substitution; import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction; +import com.google.devtools.build.lib.analysis.starlark.UnresolvedSymlinkAction; import com.google.devtools.build.lib.buildeventstream.BuildEvent; import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos; import com.google.devtools.build.lib.cmdline.RepositoryMapping; @@ -345,6 +346,13 @@ private void writeAction(ActionAnalysisMetadata action, PrintStream printStream) .append("]\n"); } + if (action instanceof UnresolvedSymlinkAction) { + stringBuilder + .append(" UnresolvedSymlinkTarget: ") + .append(((UnresolvedSymlinkAction) action).getTarget()) + .append("\n"); + } + stringBuilder.append('\n'); printStream.write(stringBuilder.toString().getBytes(UTF_8)); diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/ActionGraphDump.java b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/ActionGraphDump.java index 7e80cc27864339..0dc5c0be9322bf 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/ActionGraphDump.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/ActionGraphDump.java @@ -36,6 +36,7 @@ import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction; import com.google.devtools.build.lib.analysis.actions.TemplateExpansionException; import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget; +import com.google.devtools.build.lib.analysis.starlark.UnresolvedSymlinkAction; import com.google.devtools.build.lib.buildeventstream.BuildEvent; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.events.ExtendedEventHandler; @@ -201,6 +202,11 @@ private void dumpSingleAction(ConfiguredTarget configuredTarget, ActionAnalysisM actionBuilder.setFileContents(contents); } + if (action instanceof UnresolvedSymlinkAction) { + actionBuilder.setUnresolvedSymlinkTarget( + ((UnresolvedSymlinkAction) action).getTarget().toString()); + } + // Include the content of param files in output. if (includeParamFiles) { // Assumption: if an Action takes a params file as an input, it will be used diff --git a/src/main/protobuf/analysis_v2.proto b/src/main/protobuf/analysis_v2.proto index d3ae80b04f19dc..cfe4d76797baca 100644 --- a/src/main/protobuf/analysis_v2.proto +++ b/src/main/protobuf/analysis_v2.proto @@ -116,6 +116,10 @@ message Action { // The contents of the file for the actions.write() action // (guarded by the --include_file_write_contents flag). string file_contents = 17; + + // The target of the symlink created by UnresolvedSymlink actions. + // For regular Symlink actions, the target is represented as an input. + string unresolved_symlink_target = 18; } // Represents a single target (without configuration information) that is diff --git a/src/test/shell/integration/aquery_test.sh b/src/test/shell/integration/aquery_test.sh index 0ebf8cb8739653..3e328b31c0e0ca 100755 --- a/src/test/shell/integration/aquery_test.sh +++ b/src/test/shell/integration/aquery_test.sh @@ -1768,6 +1768,46 @@ EOF fi } +function test_unresolved_symlinks() { + local pkg="${FUNCNAME[0]}" + mkdir -p "$pkg" || fail "mkdir -p $pkg" + touch "$pkg/foo.sh" + cat > "$pkg/unresolved_symlink.bzl" <<'EOF' +def _impl(ctx): + out = ctx.actions.declare_symlink(ctx.label.name) + ctx.actions.symlink( + output = out, + target_path = ctx.attr.path + ) + return [ + DefaultInfo(files = depset([out])) + ] +unresolved_symlink = rule( + implementation = _impl, + attrs = { + "path": attr.string(), + }, +) +EOF + cat > "$pkg/BUILD" <<'EOF' +load(":unresolved_symlink.bzl", "unresolved_symlink") +unresolved_symlink( + name = "foo", + path = "bar/baz.txt", +) +EOF + bazel aquery --output=textproto \ + "//$pkg:foo" >output 2> "$TEST_log" || fail "Expected success" + cat output >> "$TEST_log" + assert_contains "^unresolved_symlink_target:.*bar/baz.txt" output + + bazel aquery --output=text "//$pkg:foo" | \ + sed -nr '/Mnemonic: UnresolvedSymlink/,/^ *$/p' >output \ + 2> "$TEST_log" || fail "Expected success" + cat output >> "$TEST_log" + assert_contains "^ *UnresolvedSymlinkTarget:.*bar/baz.txt" output +} + function test_does_not_fail_horribly_with_file() { rm -rf peach mkdir -p peach From f5d8a45e87e8a62bd1dbd134b6118c7f87e7a075 Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 12:34:55 -0700 Subject: [PATCH 43/51] Extend SkyQueryEnvironment to pass query Callback to `#beforeEvaluateQuery()`. Subclasses of SkyQueryEnvironment can inspect the `ThreadSafeOutputFormatterCallback` to optimize how they initialize query evaluation. PiperOrigin-RevId: 545756099 Change-Id: I05dd0fba4aa70e98fb07147548ff99371bf5db43 --- .../devtools/build/lib/query2/SkyQueryEnvironment.java | 10 ++++++---- .../build/lib/skyframe/PackageErrorMessageValue.java | 8 ++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/query2/SkyQueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/SkyQueryEnvironment.java index 4d7c103fdbb753..fe93375460ca1a 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/SkyQueryEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/query2/SkyQueryEnvironment.java @@ -260,7 +260,8 @@ protected EvaluationContext newEvaluationContext() { .build(); } - protected void beforeEvaluateQuery(QueryExpression expr) + protected void beforeEvaluateQuery( + QueryExpression expr, ThreadSafeOutputFormatterCallback callback) throws QueryException, InterruptedException { UniverseSkyKey universeKey = universeScope.getUniverseKey(expr, parserPrefix); ImmutableList universeScopeListToUse = universeKey.getPatterns(); @@ -295,10 +296,11 @@ protected void beforeEvaluateQuery(QueryExpression expr) /* workQueue= */ new BlockingStack<>(), new ThreadFactoryBuilder().setNameFormat("QueryEnvironment %d").build())); } - resolver = makeNewTargetPatternResolver(); + resolver = makeNewTargetPatternResolver(expr, callback); } - protected TargetPatternResolver makeNewTargetPatternResolver() { + protected TargetPatternResolver makeNewTargetPatternResolver( + QueryExpression expr, ThreadSafeOutputFormatterCallback callback) { return new RecursivePackageProviderBackedTargetPatternResolver( graphBackedRecursivePackageProvider, eventHandler, @@ -458,7 +460,7 @@ protected void handleInterruptedShutdown() {} public QueryEvalResult evaluateQuery( QueryExpression expr, ThreadSafeOutputFormatterCallback callback) throws QueryException, InterruptedException, IOException { - beforeEvaluateQuery(expr); + beforeEvaluateQuery(expr, callback); // SkyQueryEnvironment batches callback invocations using a BatchStreamedCallback, created here // so that there's one per top-level evaluateQuery call. The batch size is large enough that diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PackageErrorMessageValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/PackageErrorMessageValue.java index fa545974dd1299..54c68a635525b2 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/PackageErrorMessageValue.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/PackageErrorMessageValue.java @@ -56,7 +56,7 @@ public enum Result { * If {@code getResult().equals(NO_SUCH_PACKAGE_EXCEPTION)}, returns the error message from the * {@link com.google.devtools.build.lib.packages.NoSuchPackageException} encountered. */ - abstract String getNoSuchPackageExceptionMessage(); + public abstract String getNoSuchPackageExceptionMessage(); static PackageErrorMessageValue ofPackageWithNoErrors() { return NO_ERROR_VALUE; @@ -109,7 +109,7 @@ public Result getResult() { } @Override - String getNoSuchPackageExceptionMessage() { + public String getNoSuchPackageExceptionMessage() { throw new IllegalStateException(); } }; @@ -123,7 +123,7 @@ public Result getResult() { } @Override - String getNoSuchPackageExceptionMessage() { + public String getNoSuchPackageExceptionMessage() { throw new IllegalStateException(); } }; @@ -141,7 +141,7 @@ public Result getResult() { } @Override - String getNoSuchPackageExceptionMessage() { + public String getNoSuchPackageExceptionMessage() { return errorMessage; } From 8715e9ae6f5c2fd42607018b3adaad14bf601a1c Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Wed, 5 Jul 2023 12:35:06 -0700 Subject: [PATCH 44/51] Add toolchain type for Java bootstrap runtime The `bootclasspath` rule in `rules_java` will soon use this toolchain type instead of the regular Java runtime toolchain type, which naturally carries a constraint on the target platform. Also adds more detailed explanations of the now three Java toolchain types. Work towards https://github.com/bazelbuild/bazel/issues/17085 Work towards https://github.com/bazelbuild/bazel/issues/18265 Work towards https://github.com/bazelbuild/rules_java/issues/64 Split off from https://github.com/bazelbuild/bazel/pull/18262 Closes #18841. PiperOrigin-RevId: 545756139 Change-Id: Ib9dd7a1c20c32315375722d6a023a89859daea9c --- tools/jdk/BUILD.tools | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/tools/jdk/BUILD.tools b/tools/jdk/BUILD.tools index 37171c677c39b6..1ab0cb8e600a0a 100644 --- a/tools/jdk/BUILD.tools +++ b/tools/jdk/BUILD.tools @@ -7,12 +7,42 @@ exports_files([ "java_stub_template.txt", ]) -# Used to distinguish toolchains used for Java development, ie the JavaToolchainProvider. +# A single binary distribution of a JDK (e.g., OpenJDK 17 for Windows arm64) provides three +# different types of toolchains from the perspective of Bazel: + +# The compilation toolchain, which provides the Java runtime used to execute the Java compiler, as +# well as various helper tools and settings. +# +# Toolchains of this type typically have constraints on the execution platform so that their Java +# runtime can run the compiler, but not on the target platform as Java compilation outputs are +# platform independent. +# +# Obtain the associated JavaToolchainInfo via: +# ctx.toolchains["@bazel_tools//tools/jdk:toolchain_type"].java toolchain_type(name = "toolchain_type") -# Used to distinguish toolchains used for Java execution, ie the JavaRuntimeInfo. +# The Java runtime that executable Java compilation outputs (e.g., java_binary with +# create_executable = True) will run on. +# +# Toolchains of this type typically have constraints on the target platform so that the runtime's +# native 'java' binary can be run there, but not on the execution platform as building an executable +# Java target only requires copying or symlinking the runtime, which can be done on any platform. +# +# Obtain the associated JavaRuntimeInfo via: +# ctx.toolchains["@bazel_tools//tools/jdk:runtime_toolchain_type"].java_runtime toolchain_type(name = "runtime_toolchain_type") +# The Java runtime to extract the bootclasspath from that is then used to compile Java sources. +# +# As the bootclasspath is platform independent, toolchains of this type may have no constraints. +# Purely as an optimization to prevent unnecessary fetches of remote runtimes for other +# architectures, toolchains of this type may have constraints on the execution platform that match +# those on the corresponding compilation toolchain. +# +# Toolchains of this type are only consumed internally by the bootclasspath rule and should not be +# accessed from Starlark. +toolchain_type(name = "bootstrap_runtime_toolchain_type") + # Aliases value of --plugins flag as a JavaPluginInfo java_plugins_flag_alias( name = "java_plugins_flag_alias", From ade32e68dc7aa23b5038cd44a4b86cb491818139 Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 13:12:31 -0700 Subject: [PATCH 45/51] Add additional_compiler_inputs attribute for cc_library. Also, ensure that location function expansion works with these inputs as expected. RELNOTES: Additional source inputs can now be specified for compilation in cc_library targets using the additional_compiler_inputs attribute, and these inputs can be used in the $(location) function. Fixes #18766. PiperOrigin-RevId: 545766084 Change-Id: I2d9f195d81a1358c696601873e60d3cad810a150 --- .../bazel/rules/cpp/BazelCcLibraryRule.java | 6 + .../builtins_bzl/common/cc/cc_helper.bzl | 7 +- .../builtins_bzl/common/cc/cc_library.bzl | 5 + .../cpp/CcLibraryConfiguredTargetTest.java | 181 ++++++++++-------- 4 files changed, 114 insertions(+), 85 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcLibraryRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcLibraryRule.java index a8f3b3518e3327..ea4e63e20126e7 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcLibraryRule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCcLibraryRule.java @@ -63,6 +63,12 @@ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) attr("implementation_deps", LABEL_LIST) .allowedFileTypes(FileTypeSet.NO_FILE) .mandatoryProviders(CcInfo.PROVIDER.id())) + /* + Any additional files you might want to pass to the compiler command line, such as sanitizer + ignorelists, for example. Files specified here can then be used in copts with the + $(location) function. + */ + .add(attr("additional_compiler_inputs", LABEL_LIST).allowedFileTypes(FileTypeSet.ANY_FILE)) .advertiseStarlarkProvider(CcInfo.PROVIDER.id()) .build(); } diff --git a/src/main/starlark/builtins_bzl/common/cc/cc_helper.bzl b/src/main/starlark/builtins_bzl/common/cc/cc_helper.bzl index 094eca5acb0e84..cde737f8a9a88c 100644 --- a/src/main/starlark/builtins_bzl/common/cc/cc_helper.bzl +++ b/src/main/starlark/builtins_bzl/common/cc/cc_helper.bzl @@ -908,16 +908,19 @@ def _expand_single_make_variable(ctx, token, additional_make_variable_substituti def _expand_make_variables_for_copts(ctx, tokenization, unexpanded_tokens, additional_make_variable_substitutions): tokens = [] + targets = [] + for additional_compiler_input in getattr(ctx.attr, "additional_compiler_inputs", []): + targets.append(additional_compiler_input) for token in unexpanded_tokens: if tokenization: - expanded_token = _expand(ctx, token, additional_make_variable_substitutions) + expanded_token = _expand(ctx, token, additional_make_variable_substitutions, targets = targets) _tokenize(tokens, expanded_token) else: exp = _expand_single_make_variable(ctx, token, additional_make_variable_substitutions) if exp != None: _tokenize(tokens, exp) else: - tokens.append(_expand(ctx, token, additional_make_variable_substitutions)) + tokens.append(_expand(ctx, token, additional_make_variable_substitutions, targets = targets)) return tokens def _get_copts(ctx, feature_configuration, additional_make_variable_substitutions): diff --git a/src/main/starlark/builtins_bzl/common/cc/cc_library.bzl b/src/main/starlark/builtins_bzl/common/cc/cc_library.bzl index 581237a257ab14..5e1e2e185cb49c 100755 --- a/src/main/starlark/builtins_bzl/common/cc/cc_library.bzl +++ b/src/main/starlark/builtins_bzl/common/cc/cc_library.bzl @@ -73,6 +73,7 @@ def _cc_library_impl(ctx): textual_hdrs = ctx.files.textual_hdrs, include_prefix = ctx.attr.include_prefix, strip_include_prefix = ctx.attr.strip_include_prefix, + additional_inputs = ctx.files.additional_compiler_inputs, ) precompiled_objects = cc_common.create_compilation_outputs( @@ -592,6 +593,10 @@ attrs = { "win_def_file": attr.label(allow_single_file = [".def"]), # buildifier: disable=attr-license "licenses": attr.license() if hasattr(attr, "license") else attr.string_list(), + "additional_compiler_inputs": attr.label_list( + allow_files = True, + flags = ["ORDER_INDEPENDENT", "DIRECT_COMPILE_TIME_INPUT"], + ), "_stl": semantics.get_stl(), "_grep_includes": attr.label( allow_files = True, diff --git a/src/test/java/com/google/devtools/build/lib/rules/cpp/CcLibraryConfiguredTargetTest.java b/src/test/java/com/google/devtools/build/lib/rules/cpp/CcLibraryConfiguredTargetTest.java index 546befb6575a87..6247c7ee2042e3 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/cpp/CcLibraryConfiguredTargetTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/cpp/CcLibraryConfiguredTargetTest.java @@ -54,9 +54,7 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -/** - * "White-box" unit test of cc_library rule. - */ +/** "White-box" unit test of cc_library rule. */ @RunWith(JUnit4.class) public class CcLibraryConfiguredTargetTest extends BuildViewTestCase { private static final PathFragment STL_CPPMAP = PathFragment.create("stl_cc_library.cppmap"); @@ -108,9 +106,10 @@ private CppCompileAction getCppCompileAction(String label) throws Exception { private CppCompileAction getCppCompileAction(ConfiguredTarget target) throws Exception { List compilationSteps = - actionsTestUtil().findTransitivePrerequisitesOf( - ActionsTestUtil.getFirstArtifactEndingWith(getFilesToBuild(target), ".a"), - CppCompileAction.class); + actionsTestUtil() + .findTransitivePrerequisitesOf( + ActionsTestUtil.getFirstArtifactEndingWith(getFilesToBuild(target), ".a"), + CppCompileAction.class); return compilationSteps.get(0); } @@ -150,9 +149,12 @@ public void checkWrongExtensionInArtifactNamePattern( @Test public void testDefinesAndMakeVariables() throws Exception { - ConfiguredTarget l = scratchConfiguredTarget("a", "l", - "cc_library(name='l', srcs=['l.cc'], defines=['V=$(FOO)'], toolchains=[':v'])", - "make_variable_tester(name='v', variables={'FOO': 'BAR'})"); + ConfiguredTarget l = + scratchConfiguredTarget( + "a", + "l", + "cc_library(name='l', srcs=['l.cc'], defines=['V=$(FOO)'], toolchains=[':v'])", + "make_variable_tester(name='v', variables={'FOO': 'BAR'})"); assertThat(l.get(CcInfo.PROVIDER).getCcCompilationContext().getDefines()).contains("V=BAR"); } @@ -504,15 +506,13 @@ public void testWindowsFileNamePatternsCanBeSpecifiedInToolchain() throws Except @Test public void testWrongObjectFileArtifactNamePattern() throws Exception { checkWrongExtensionInArtifactNamePattern( - "object_file", - ArtifactCategory.OBJECT_FILE.getAllowedExtensions()); + "object_file", ArtifactCategory.OBJECT_FILE.getAllowedExtensions()); } @Test public void testWrongStaticLibraryArtifactNamePattern() throws Exception { checkWrongExtensionInArtifactNamePattern( - "static_library", - ArtifactCategory.STATIC_LIBRARY.getAllowedExtensions()); + "static_library", ArtifactCategory.STATIC_LIBRARY.getAllowedExtensions()); } @Test @@ -525,22 +525,19 @@ public void testWrongAlwayslinkStaticLibraryArtifactNamePattern() throws Excepti @Test public void testWrongExecutableArtifactNamePattern() throws Exception { checkWrongExtensionInArtifactNamePattern( - "executable", - ArtifactCategory.EXECUTABLE.getAllowedExtensions()); + "executable", ArtifactCategory.EXECUTABLE.getAllowedExtensions()); } @Test public void testWrongDynamicLibraryArtifactNamePattern() throws Exception { checkWrongExtensionInArtifactNamePattern( - "dynamic_library", - ArtifactCategory.DYNAMIC_LIBRARY.getAllowedExtensions()); + "dynamic_library", ArtifactCategory.DYNAMIC_LIBRARY.getAllowedExtensions()); } @Test public void testWrongInterfaceLibraryArtifactNamePattern() throws Exception { checkWrongExtensionInArtifactNamePattern( - "interface_library", - ArtifactCategory.INTERFACE_LIBRARY.getAllowedExtensions()); + "interface_library", ArtifactCategory.INTERFACE_LIBRARY.getAllowedExtensions()); } @Test @@ -591,11 +588,13 @@ public void testTransitiveArtifactsToAlwaysBuildStatic() throws Exception { mockToolsConfig, CcToolchainConfig.builder().withFeatures(CppRuleClasses.SUPPORTS_PIC)); useConfiguration("--cpu=k8"); - ConfiguredTarget x = scratchConfiguredTarget( - "foo", "x", - "cc_library(name = 'x', srcs = ['x.cc'], deps = [':y'], linkstatic = 1)", - "cc_library(name = 'y', srcs = ['y.cc'], deps = [':z'])", - "cc_library(name = 'z', srcs = ['z.cc'])"); + ConfiguredTarget x = + scratchConfiguredTarget( + "foo", + "x", + "cc_library(name = 'x', srcs = ['x.cc'], deps = [':y'], linkstatic = 1)", + "cc_library(name = 'y', srcs = ['y.cc'], deps = [':z'])", + "cc_library(name = 'z', srcs = ['z.cc'])"); assertThat(artifactsToStrings(getOutputGroup(x, OutputGroupInfo.HIDDEN_TOP_LEVEL))) .containsExactly( "bin foo/_objs/x/x.pic.o", "bin foo/_objs/y/y.pic.o", "bin foo/_objs/z/z.pic.o"); @@ -653,13 +652,15 @@ public void testDisablingHeaderModulesWhenDependingOnModuleBuildTransitively() t mockToolsConfig, CcToolchainConfig.builder().withFeatures(MockCcSupport.HEADER_MODULES_FEATURES)); useConfiguration(); - scratch.file("module/BUILD", + scratch.file( + "module/BUILD", "package(features = ['header_modules'])", "cc_library(", " name = 'module',", " srcs = ['a.cc', 'a.h'],", ")"); - scratch.file("nomodule/BUILD", + scratch.file( + "nomodule/BUILD", "package(features = ['-header_modules'])", "cc_library(", " name = 'nomodule',", @@ -690,9 +691,7 @@ private static Iterable getHeaderModules(NestedSet input) { input.toList(), (artifact) -> CppFileTypes.CPP_MODULE.matches(artifact.getExecPath())); } - /** - * Returns the flags in {@code input} that reference a header module. - */ + /** Returns the flags in {@code input} that reference a header module. */ private Iterable getHeaderModuleFlags(Iterable input) { List names = new ArrayList<>(); for (String flag : input) { @@ -726,16 +725,16 @@ public void testCompileHeaderModules() throws Exception { ConfiguredTarget moduleB = getConfiguredTarget("//module:b"); Artifact bModuleArtifact = getBinArtifact("_objs/b/b.pic.pcm", moduleB); CppCompileAction bModuleAction = (CppCompileAction) getGeneratingAction(bModuleArtifact); - assertThat(bModuleAction.getIncludeScannerSources()).containsExactly( - getSourceArtifact("module/b.h"), getSourceArtifact("module/t.h")); + assertThat(bModuleAction.getIncludeScannerSources()) + .containsExactly(getSourceArtifact("module/b.h"), getSourceArtifact("module/t.h")); assertThat(bModuleAction.getInputs().toList()) .contains(getGenfilesArtifact("b.cppmap", moduleB)); ConfiguredTarget moduleA = getConfiguredTarget("//module:a"); Artifact aObjectArtifact = getBinArtifact("_objs/a/a.pic.o", moduleA); CppCompileAction aObjectAction = (CppCompileAction) getGeneratingAction(aObjectArtifact); - assertThat(aObjectAction.getIncludeScannerSources()).containsExactly( - getSourceArtifact("module/a.cc")); + assertThat(aObjectAction.getIncludeScannerSources()) + .containsExactly(getSourceArtifact("module/a.cc")); assertThat(aObjectAction.getCcCompilationContext().getTransitiveModules(true).toList()) .contains(getBinArtifact("_objs/b/b.pic.pcm", moduleB)); assertThat(aObjectAction.getInputs().toList()) @@ -788,7 +787,8 @@ public void testContainingSourcesWithSameBaseName() throws Exception { } private void setupPackagesForModuleTests(boolean useHeaderModules) throws Exception { - scratch.file("module/BUILD", + scratch.file( + "module/BUILD", "package(features = ['header_modules'])", "cc_library(", " name = 'b',", @@ -805,9 +805,11 @@ private void setupPackagesForModuleTests(boolean useHeaderModules) throws Except " srcs = ['j.h', 'j.cc'],", " deps = ['//nomodule:c', '//nomodule:i'],", ")"); - scratch.file("nomodule/BUILD", + scratch.file( + "nomodule/BUILD", "package(features = ['-header_modules'" - + (useHeaderModules ? ", 'use_header_modules'" : "") + "])", + + (useHeaderModules ? ", 'use_header_modules'" : "") + + "])", "cc_library(", " name = 'y',", " srcs = ['y.h'],", @@ -852,7 +854,7 @@ private void setupPackagesForModuleTests(boolean useHeaderModules) throws Except " srcs = ['i.h', 'i.cc'],", " deps = [':h'],", ")"); - } + } @Test public void testCompileHeaderModulesTransitively() throws Exception { @@ -876,8 +878,8 @@ public void testCompileHeaderModulesTransitively() throws Exception { .containsExactly( getGenfilesArtifact("f.cppmap", nomoduleF), getGenfilesArtifact("e.cppmap", nomoduleE)); assertThat(getHeaderModules(fObjectAction.getInputs())).isEmpty(); - assertThat(fObjectAction.getIncludeScannerSources()).containsExactly( - getSourceArtifact("nomodule/f.cc")); + assertThat(fObjectAction.getIncludeScannerSources()) + .containsExactly(getSourceArtifact("nomodule/f.cc")); assertThat(getHeaderModuleFlags(fObjectAction.getCompilerOptions())).isEmpty(); // The //nomodule:c target will get the header module for //module:b, which is a direct @@ -893,10 +895,10 @@ public void testCompileHeaderModulesTransitively() throws Exception { assertThat(getHeaderModules(cObjectAction.getInputs())).isEmpty(); // All headers of transitive dependencies that are built as modules are needed as entry points // for include scanning. - assertThat(cObjectAction.getIncludeScannerSources()).containsExactly( - getSourceArtifact("nomodule/c.cc")); - assertThat(cObjectAction.getMainIncludeScannerSource()).isEqualTo( - getSourceArtifact("nomodule/c.cc")); + assertThat(cObjectAction.getIncludeScannerSources()) + .containsExactly(getSourceArtifact("nomodule/c.cc")); + assertThat(cObjectAction.getMainIncludeScannerSource()) + .isEqualTo(getSourceArtifact("nomodule/c.cc")); assertThat(getHeaderModuleFlags(cObjectAction.getCompilerOptions())).isEmpty(); // The //nomodule:d target depends on //module:b via one indirection (//nomodule:c). @@ -909,12 +911,13 @@ public void testCompileHeaderModulesTransitively() throws Exception { getBinArtifact("_objs/d/d.pic.o", getConfiguredTarget("//nomodule:d")); CppCompileAction dObjectAction = (CppCompileAction) getGeneratingAction(dObjectArtifact); // Module map 'c.cppmap' is needed because it is a direct dependency. - assertThat(getNonSystemModuleMaps(dObjectAction.getInputs())).containsExactly( - getGenfilesArtifact("c.cppmap", "//nomodule:c"), - getGenfilesArtifact("d.cppmap", "//nomodule:d")); + assertThat(getNonSystemModuleMaps(dObjectAction.getInputs())) + .containsExactly( + getGenfilesArtifact("c.cppmap", "//nomodule:c"), + getGenfilesArtifact("d.cppmap", "//nomodule:d")); assertThat(getHeaderModules(dObjectAction.getInputs())).isEmpty(); - assertThat(dObjectAction.getIncludeScannerSources()).containsExactly( - getSourceArtifact("nomodule/d.cc")); + assertThat(dObjectAction.getIncludeScannerSources()) + .containsExactly(getSourceArtifact("nomodule/d.cc")); assertThat(getHeaderModuleFlags(dObjectAction.getCompilerOptions())).isEmpty(); // The //module:j target depends on //module:g via //nomodule:h and on //module:b via @@ -926,10 +929,10 @@ public void testCompileHeaderModulesTransitively() throws Exception { .containsExactly( getBinArtifact("_objs/b/b.pic.pcm", getConfiguredTarget("//module:b")), getBinArtifact("_objs/g/g.pic.pcm", getConfiguredTarget("//module:g"))); - assertThat(jObjectAction.getIncludeScannerSources()).containsExactly( - getSourceArtifact("module/j.cc")); - assertThat(jObjectAction.getMainIncludeScannerSource()).isEqualTo( - getSourceArtifact("module/j.cc")); + assertThat(jObjectAction.getIncludeScannerSources()) + .containsExactly(getSourceArtifact("module/j.cc")); + assertThat(jObjectAction.getMainIncludeScannerSource()) + .isEqualTo(getSourceArtifact("module/j.cc")); assertThat(getHeaderModules(jObjectAction.getCcCompilationContext().getTransitiveModules(true))) .containsExactly( getBinArtifact("_objs/b/b.pic.pcm", getConfiguredTarget("//module:b")), @@ -982,11 +985,8 @@ public void testCompileUsingHeaderModulesTransitively() throws Exception { } private void writeSimpleCcLibrary() throws Exception { - scratch.file("module/BUILD", - "cc_library(", - " name = 'map',", - " srcs = ['a.cc', 'a.h'],", - ")"); + scratch.file( + "module/BUILD", "cc_library(", " name = 'map',", " srcs = ['a.cc', 'a.h'],", ")"); } @Test @@ -1005,7 +1005,9 @@ public void testToolchainWithoutPicForNoPicCompilation() throws Exception { CppActionNames.CPP_LINK_STATIC_LIBRARY, CppActionNames.STRIP)); useConfiguration("--features=-supports_pic"); - scratchConfiguredTarget("a", "a", + scratchConfiguredTarget( + "a", + "a", "cc_binary(name='a', srcs=['a.cc'], deps=[':b'])", "cc_library(name='b', srcs=['b.cc'])"); } @@ -1040,8 +1042,7 @@ public void testCppModuleMap() throws Exception { CppModuleMapAction action = getCppModuleMapAction("//module:map"); assertThat(ActionsTestUtil.baseArtifactNames(action.getDependencyArtifacts())) .contains("crosstool.cppmap"); - assertThat(artifactsToStrings(action.getPrivateHeaders())) - .containsExactly("src module/a.h"); + assertThat(artifactsToStrings(action.getPrivateHeaders())).containsExactly("src module/a.h"); assertThat(action.getPublicHeaders()).isEmpty(); } @@ -1051,9 +1052,12 @@ public void testCppModuleMap() throws Exception { */ @Test public void testFilesToBuildWithPrecompiledStaticLibrary() throws Exception { - ConfiguredTarget hello = scratchConfiguredTarget("precompiled", "library", - "cc_library(name = 'library', ", - " srcs = ['missing.a'])"); + ConfiguredTarget hello = + scratchConfiguredTarget( + "precompiled", + "library", + "cc_library(name = 'library', ", + " srcs = ['missing.a'])"); assertThat(artifactsToStrings(getFilesToBuild(hello))) .doesNotContain("src precompiled/missing.a"); } @@ -1224,7 +1228,8 @@ private CppCompileAction getGeneratingCompileAction( @Test public void testIncludePathOrder() throws Exception { useConfiguration("--incompatible_merge_genfiles_directory=false"); - scratch.file("foo/BUILD", + scratch.file( + "foo/BUILD", "cc_library(", " name = 'bar',", " includes = ['bar'],", @@ -1260,7 +1265,8 @@ public void testIncludePathOrder() throws Exception { @Test public void testDefinesOrder() throws Exception { - scratch.file("foo/BUILD", + scratch.file( + "foo/BUILD", "cc_library(", " name = 'bar',", " defines = ['BAR'],", @@ -1303,7 +1309,9 @@ public void testLocalDefinesNotPassedTransitively() throws Exception { // Regression test - setting "-shared" caused an exception when computing the link command. @Test public void testLinkOptsNotPassedToStaticLink() throws Exception { - scratchConfiguredTarget("foo", "foo", + scratchConfiguredTarget( + "foo", + "foo", "cc_library(", " name = 'foo',", " srcs = ['foo.cc'],", @@ -1352,10 +1360,7 @@ public void testCompilationModeFeatures() throws Exception { @Test public void testIncludePathsOutsideExecutionRoot() throws Exception { - scratchRule( - "root", - "a", - "cc_library(name='a', srcs=['a.cc'], copts=['-Id/../../somewhere'])"); + scratchRule("root", "a", "cc_library(name='a', srcs=['a.cc'], copts=['-Id/../../somewhere'])"); CppCompileAction compileAction = getCppCompileAction("//root:a"); try { compileAction.verifyActionIncludePaths(compileAction.getSystemIncludeDirs(), false); @@ -1369,10 +1374,7 @@ public void testIncludePathsOutsideExecutionRoot() throws Exception { @Test public void testAbsoluteIncludePathsOutsideExecutionRoot() throws Exception { - scratchRule( - "root", - "a", - "cc_library(name='a', srcs=['a.cc'], copts=['-I/somewhere'])"); + scratchRule("root", "a", "cc_library(name='a', srcs=['a.cc'], copts=['-I/somewhere'])"); CppCompileAction compileAction = getCppCompileAction("//root:a"); try { compileAction.verifyActionIncludePaths(compileAction.getSystemIncludeDirs(), false); @@ -1386,10 +1388,7 @@ public void testAbsoluteIncludePathsOutsideExecutionRoot() throws Exception { @Test public void testSystemIncludePathsOutsideExecutionRoot() throws Exception { - scratchRule( - "root", - "a", - "cc_library(name='a', srcs=['a.cc'], copts=['-isystem../system'])"); + scratchRule("root", "a", "cc_library(name='a', srcs=['a.cc'], copts=['-isystem../system'])"); CppCompileAction compileAction = getCppCompileAction("//root:a"); try { compileAction.verifyActionIncludePaths(compileAction.getSystemIncludeDirs(), false); @@ -1403,10 +1402,7 @@ public void testSystemIncludePathsOutsideExecutionRoot() throws Exception { @Test public void testAbsoluteSystemIncludePathsOutsideExecutionRoot() throws Exception { - scratchRule( - "root", - "a", - "cc_library(name='a', srcs=['a.cc'], copts=['-isystem/system'])"); + scratchRule("root", "a", "cc_library(name='a', srcs=['a.cc'], copts=['-isystem/system'])"); CppCompileAction compileAction = getCppCompileAction("//root:a"); try { compileAction.verifyActionIncludePaths(compileAction.getSystemIncludeDirs(), false); @@ -1557,7 +1553,6 @@ public void forbidBuildingAndWrappingSameLibraryIdentifier() throws Exception { "cc_library(name = 'foo', srcs = ['foo.cc', 'libfoo.lo'])"); } - @Test public void testProcessedHeadersWithPicSharedLibsAndNoPicBinaries() throws Exception { AnalysisMock.get() @@ -2408,4 +2403,24 @@ public void testDataDepRunfilesArePropagated() throws Exception { lib.get(DefaultInfo.PROVIDER).getDefaultRunfiles().getAllArtifacts())) .containsExactly("src foo/file.txt"); } + + @Test + public void testAdditionalCompilerInputsArePassedToCompile() throws Exception { + AnalysisMock.get().ccSupport().setupCcToolchainConfig(mockToolsConfig); + scratch.file( + "foo/BUILD", + "cc_library(", + " name = 'foo',", + " srcs = ['hello.cc'],", + " copts = ['$(location compiler_input.txt)'],", + " additional_compiler_inputs = ['compiler_input.txt'],", + ")"); + scratch.file("foo/compiler_input.txt", "hello world!"); + + ConfiguredTarget lib = getConfiguredTarget("//foo:foo"); + Artifact artifact = getBinArtifact("_objs/foo/hello.o", lib); + CppCompileAction action = (CppCompileAction) getGeneratingAction(artifact); + assertThat(action.getInputs().toList()).contains(getSourceArtifact("foo/compiler_input.txt")); + assertThat(action.getArguments()).contains("foo/compiler_input.txt"); + } } From f53a29201efa99bf67f353be5595dcd0c8f4c17a Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 13:22:31 -0700 Subject: [PATCH 46/51] Implement `MessageBundleInfo` provider in Starlark PiperOrigin-RevId: 545768807 Change-Id: Iec8a0e4114310a11927816f42e99d3f361c87de2 --- .../build/lib/rules/java/JavaCommon.java | 2 +- .../lib/rules/java/JavaStarlarkCommon.java | 7 -- .../lib/rules/java/MessageBundleInfo.java | 74 +++++++------------ .../starlarkbuildapi/java/JavaCommonApi.java | 7 -- .../java/MessageBundleInfoApi.java | 27 ------- .../builtins_bzl/common/java/java_common.bzl | 3 +- .../common/java/message_bundle_info.bzl | 24 ++++++ 7 files changed, 53 insertions(+), 91 deletions(-) delete mode 100644 src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/MessageBundleInfoApi.java create mode 100644 src/main/starlark/builtins_bzl/common/java/message_bundle_info.bzl diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCommon.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCommon.java index 5b40d3f2081487..29ea5698421245 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCommon.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCommon.java @@ -674,7 +674,7 @@ public void addGenJarsProvider( } /** Processes the sources of this target, adding them as messages or proper sources. */ - private void processSrcs(JavaTargetAttributes.Builder attributes) { + private void processSrcs(JavaTargetAttributes.Builder attributes) throws RuleErrorException { List srcs = ruleContext.getPrerequisites("srcs"); for (TransitiveInfoCollection src : srcs) { ImmutableList messages = MessageBundleInfo.getMessages(src); diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java index 6f1997e40803a4..1b9741db719bd3 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java @@ -276,13 +276,6 @@ public Provider getJavaRuntimeProvider() { return JavaRuntimeInfo.PROVIDER; } - @Override - public ProviderApi getMessageBundleInfo() { - // No implementation in Bazel. This method not callable in Starlark except through - // (discouraged) use of --experimental_google_legacy_api. - return null; - } - @Override public JavaInfo setAnnotationProcessing( Info info, diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/MessageBundleInfo.java b/src/main/java/com/google/devtools/build/lib/rules/java/MessageBundleInfo.java index 61c2e1ca083027..a98c86edc5ca6e 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/MessageBundleInfo.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/MessageBundleInfo.java @@ -14,83 +14,61 @@ package com.google.devtools.build.lib.rules.java; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; -import com.google.devtools.build.lib.packages.BuiltinProvider; -import com.google.devtools.build.lib.packages.NativeInfo; -import com.google.devtools.build.lib.starlarkbuildapi.core.ProviderApi; -import com.google.devtools.build.lib.starlarkbuildapi.java.MessageBundleInfoApi; -import java.util.List; +import com.google.devtools.build.lib.packages.Info; +import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; +import com.google.devtools.build.lib.packages.StarlarkInfo; +import com.google.devtools.build.lib.packages.StarlarkProviderWrapper; import javax.annotation.Nullable; -import net.starlark.java.annot.Param; -import net.starlark.java.annot.StarlarkBuiltin; -import net.starlark.java.annot.StarlarkMethod; import net.starlark.java.eval.EvalException; import net.starlark.java.eval.Sequence; -import net.starlark.java.eval.StarlarkList; -import net.starlark.java.eval.StarlarkThread; -import net.starlark.java.syntax.Location; /** Marks configured targets that are able to supply message bundles to their dependents. */ @Immutable -public final class MessageBundleInfo extends NativeInfo implements MessageBundleInfoApi { +public final class MessageBundleInfo { - public static final String STARLARK_NAME = "MessageBundleInfo"; + private static final String STARLARK_NAME = "MessageBundleInfo"; /** Provider singleton constant. */ - public static final BuiltinProvider PROVIDER = new Provider(); + private static final StarlarkProviderWrapper PROVIDER = new Provider(); /** Provider class for {@link MessageBundleInfo} objects. */ - @StarlarkBuiltin(name = "Provider", documented = false, doc = "") - public static class Provider extends BuiltinProvider implements ProviderApi { + private static class Provider extends StarlarkProviderWrapper { private Provider() { - super(STARLARK_NAME, MessageBundleInfo.class); + super( + Label.parseCanonicalUnchecked("@_builtins//:common/java/message_bundle_info.bzl"), + STARLARK_NAME); } - @StarlarkMethod( - name = "MessageBundleInfo", - doc = "The MessageBundleInfo constructor.", - documented = false, - parameters = { - @Param(name = "messages", positional = false, named = true), - }, - selfCall = true, - useStarlarkThread = true) - public MessageBundleInfo messageBundleInfo(Sequence messages, StarlarkThread thread) - throws EvalException { - List messagesList = Sequence.cast(messages, Artifact.class, "messages"); - return new MessageBundleInfo(ImmutableList.copyOf(messagesList), thread.getCallerLocation()); + @Override + public MessageBundleInfo wrap(Info value) throws RuleErrorException { + try { + return new MessageBundleInfo((StarlarkInfo) value); + } catch (EvalException e) { + throw new RuleErrorException(e); + } } } private final ImmutableList messages; - private MessageBundleInfo(ImmutableList messages, Location creationLocation) { - super(creationLocation); - this.messages = Preconditions.checkNotNull(messages); + private MessageBundleInfo(StarlarkInfo value) throws EvalException { + this.messages = + Sequence.cast(value.getValue("messages"), Artifact.class, "messages").getImmutableList(); } - @Override - public BuiltinProvider getProvider() { - return PROVIDER; - } - - @Override - public Sequence getMessageBundles() { - return StarlarkList.immutableCopyOf(getMessages()); - } - - public ImmutableList getMessages() { + private ImmutableList getMessages() { return messages; } @Nullable - public static ImmutableList getMessages(TransitiveInfoCollection info) { - MessageBundleInfo messageBundleInfo = - (MessageBundleInfo) info.get(MessageBundleInfo.PROVIDER.getKey()); + public static ImmutableList getMessages(TransitiveInfoCollection info) + throws RuleErrorException { + MessageBundleInfo messageBundleInfo = info.get(MessageBundleInfo.PROVIDER); if (messageBundleInfo != null) { return messageBundleInfo.getMessages(); } diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java index d5a6f7cc3a83d2..41ca4264c8f154 100644 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java @@ -464,13 +464,6 @@ JavaInfoT mergeJavaProviders( structField = true) ProviderApi getJavaRuntimeProvider(); - @StarlarkMethod( - name = "MessageBundleInfo", - doc = "The provider used to supply message bundles for translation", - structField = true, - enableOnlyWithFlag = BuildLanguageOptions.EXPERIMENTAL_GOOGLE_LEGACY_API) - ProviderApi getMessageBundleInfo(); - @StarlarkMethod( name = "set_annotation_processing", doc = "Returns a copy of the given JavaInfo with the given annotation_processing info.", diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/MessageBundleInfoApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/MessageBundleInfoApi.java deleted file mode 100644 index ab4c476e9d60a6..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/MessageBundleInfoApi.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2018 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.starlarkbuildapi.java; - -import com.google.devtools.build.lib.starlarkbuildapi.FileApi; -import net.starlark.java.annot.StarlarkMethod; -import net.starlark.java.eval.Sequence; -import net.starlark.java.eval.StarlarkValue; - -/** Info object encapsulating message bundles in java rules. */ -public interface MessageBundleInfoApi extends StarlarkValue { - - @StarlarkMethod(name = "messages", structField = true, doc = "Sequence of message bundles") - Sequence getMessageBundles(); -} diff --git a/src/main/starlark/builtins_bzl/common/java/java_common.bzl b/src/main/starlark/builtins_bzl/common/java/java_common.bzl index 5a8a89f338c736..848f25c59e1bc4 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_common.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_common.bzl @@ -19,6 +19,7 @@ load( "JavaInfo", _java_info_add_constraints = "add_constraints", ) +load(":common/java/message_bundle_info.bzl", "MessageBundleInfo") load(":common/java/java_common_internal_for_builtins.bzl", "compile", "merge", "run_ijar") load(":common/java/java_plugin_info.bzl", "JavaPluginInfo") load(":common/java/java_semantics.bzl", "semantics") @@ -174,7 +175,7 @@ def _make_non_strict(java_info): return _java_common_internal.make_non_strict(java_info) def _get_message_bundle_info(): - return None if semantics.IS_BAZEL else _java_common_internal.MessageBundleInfo + return None if semantics.IS_BAZEL else MessageBundleInfo def _add_constraints(java_info, constraints = []): """Returns a copy of the given JavaInfo with the given constraints added. diff --git a/src/main/starlark/builtins_bzl/common/java/message_bundle_info.bzl b/src/main/starlark/builtins_bzl/common/java/message_bundle_info.bzl new file mode 100644 index 00000000000000..49948fa42d57ec --- /dev/null +++ b/src/main/starlark/builtins_bzl/common/java/message_bundle_info.bzl @@ -0,0 +1,24 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Definition of MessageBundleInfo provider. +""" + +MessageBundleInfo = provider( + doc = "Marks configured targets that are able to supply message bundles to their dependents.", + fields = { + "messages": "Sequence of message bundles", + }, +) From f551a92e1af72b92be1606fa295a818ce9eab256 Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 13:57:12 -0700 Subject: [PATCH 47/51] Add support for resources in `java_helper.create_single_jar` action This will be used to register `JavaSourceJar` actions in upcoming changes. Code is transliterated from `SingleJarActionBuilder` and `JavaSemantics.getDefaultJavaResourcePath` PiperOrigin-RevId: 545778459 Change-Id: Ic08812bea3e966329e961e8f7a757b6acfe3cb9e --- .../builtins_bzl/common/java/java_binary.bzl | 15 +++---- .../builtins_bzl/common/java/java_helper.bzl | 42 +++++++++++++------ .../common/java/java_semantics.bzl | 10 +++++ 3 files changed, 47 insertions(+), 20 deletions(-) diff --git a/src/main/starlark/builtins_bzl/common/java/java_binary.bzl b/src/main/starlark/builtins_bzl/common/java/java_binary.bzl index ad8d448b4b4002..1d7d4f97a39c67 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_binary.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_binary.bzl @@ -340,10 +340,10 @@ def _create_shared_archive(ctx, java_attrs): jsa = ctx.actions.declare_file("%s.jsa" % ctx.label.name) merged = ctx.actions.declare_file(jsa.dirname + "/" + helper.strip_extension(jsa) + "-merged.jar") helper.create_single_jar( - ctx, - merged, - java_attrs.runtime_jars, - java_attrs.runtime_classpath_for_archive, + ctx.actions, + toolchain = semantics.find_java_toolchain(ctx), + output = merged, + sources = depset(transitive = [java_attrs.runtime_jars, java_attrs.runtime_classpath_for_archive]), ) args = ctx.actions.args() @@ -417,9 +417,10 @@ def _create_one_version_check(ctx, inputs): def _create_deploy_sources_jar(ctx, sources): helper.create_single_jar( - ctx, - ctx.outputs.deploysrcjar, - sources, + ctx.actions, + toolchain = semantics.find_java_toolchain(ctx), + output = ctx.outputs.deploysrcjar, + sources = sources, ) def _filter_validation_output_group(ctx, output_group): diff --git a/src/main/starlark/builtins_bzl/common/java/java_helper.bzl b/src/main/starlark/builtins_bzl/common/java/java_helper.bzl index eebc50d57dec67..7c751f4e8ff3fd 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_helper.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_helper.bzl @@ -288,19 +288,35 @@ def _test_providers(ctx): return test_providers -def _create_single_jar(ctx, output, *input_depsets): - """Register action for the output jar. +def _resource_mapper(file): + return "%s:%s" % ( + file.path, + semantics.get_default_resource_path(file.short_path, segment_extractor = _java_segments), + ) + +def _create_single_jar( + actions, + toolchain, + output, + sources = depset(), + resources = depset(), + mnemonic = "JavaSingleJar", + progress_message = "Building singlejar jar %{output}"): + """Register singlejar action for the output jar. Args: - ctx: (RuleContext) Used to register the action. - output: (Artifact) Output file of the action. - *input_depsets: (list[depset[Artifact]]) Input files of the action. + actions: (actions) ctx.actions + toolchain: (JavaToolchainInfo) The java toolchain + output: (File) Output file of the action. + sources: (depset[File]) The jar files to merge into the output jar. + resources: (depset[File]) The files to add to the output jar. + mnemonic: (str) The action identifier + progress_message: (str) The action progress message Returns: (File) Output file which was used for registering the action. """ - toolchain = semantics.find_java_toolchain(ctx) - args = ctx.actions.args() + args = actions.args() args.set_param_file_format("shell").use_param_file("@%s", use_always = True) args.add("--output", output) args.add_all( @@ -311,15 +327,15 @@ def _create_single_jar(ctx, output, *input_depsets): "--warn_duplicate_resources", ], ) - all_inputs = depset(transitive = input_depsets) - args.add_all("--sources", all_inputs) - ctx.actions.run( - mnemonic = "JavaSingleJar", - progress_message = "Building singlejar jar %s" % output.short_path, + args.add_all("--sources", sources) + args.add_all("--resources", resources, map_each = _resource_mapper) + actions.run( + mnemonic = mnemonic, + progress_message = progress_message, executable = toolchain.single_jar, toolchain = semantics.JAVA_TOOLCHAIN_TYPE, - inputs = all_inputs, + inputs = depset(transitive = [resources, sources]), tools = [toolchain.single_jar], outputs = [output], arguments = [args], diff --git a/src/main/starlark/builtins_bzl/common/java/java_semantics.bzl b/src/main/starlark/builtins_bzl/common/java/java_semantics.bzl index 07686bedb09e06..a369ee6136b804 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_semantics.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_semantics.bzl @@ -28,6 +28,15 @@ def _find_java_toolchain(ctx): def _find_java_runtime_toolchain(ctx): return ctx.toolchains["@bazel_tools//tools/jdk:runtime_toolchain_type"].java_runtime +def _get_default_resource_path(path, segment_extractor): + # Look for src/.../resources to match Maven repository structure. + segments = path.split("/") + for idx in range(0, len(segments) - 2): + if segments[idx] == "src" and segments[idx + 2] == "resources": + return "/".join(segments[idx + 3:]) + java_segments = segment_extractor(path) + return "/".join(java_segments) if java_segments != None else path + semantics = struct( JAVA_TOOLCHAIN_LABEL = "@bazel_tools//tools/jdk:current_java_toolchain", JAVA_TOOLCHAIN_TYPE = "@bazel_tools//tools/jdk:toolchain_type", @@ -58,4 +67,5 @@ semantics = struct( BUILD_INFO_TRANSLATOR_LABEL = None, JAVA_TEST_RUNNER_LABEL = "@bazel_tools//tools/jdk:TestRunner", IS_BAZEL = True, + get_default_resource_path = _get_default_resource_path, ) From 6da7f21c427850e47dbada0e05ed690866608dd2 Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 14:21:30 -0700 Subject: [PATCH 48/51] Implement `java_common.pack_sources` in Starlark and delete the corresponding native API An earlier change[^1] already dropped the deprecated `output_jar` parameter so we make the `output_source_jar` parameter mandatory. PiperOrigin-RevId: 545785281 Change-Id: I96f928940b9fa3b6752f0300a3768f0e15e587b2 --- .../lib/rules/java/JavaInfoBuildHelper.java | 45 ----------- .../lib/rules/java/JavaStarlarkCommon.java | 30 ------- .../starlarkbuildapi/java/JavaCommonApi.java | 78 ------------------- .../builtins_bzl/common/java/java_common.bzl | 20 ++--- .../lib/analysis/AutoExecGroupsTest.java | 7 +- 5 files changed, 15 insertions(+), 165 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfoBuildHelper.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfoBuildHelper.java index 133a0d4c135c45..a5bdde6a54261b 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfoBuildHelper.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaInfoBuildHelper.java @@ -22,11 +22,9 @@ import com.google.common.base.Ascii; import com.google.common.collect.ImmutableList; import com.google.common.collect.Streams; -import com.google.devtools.build.lib.actions.ActionRegistry; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.analysis.actions.ActionConstructionContext; import com.google.devtools.build.lib.analysis.config.CoreOptionConverters.StrictDepsMode; -import com.google.devtools.build.lib.analysis.starlark.StarlarkActionFactory; import com.google.devtools.build.lib.analysis.starlark.StarlarkRuleContext; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; @@ -64,49 +62,6 @@ private static ImmutableList collectJavaGenJarsProviders( return builder.build(); } - /** - * Creates action which creates archive with all source files inside. Takes all filer from - * sourceFiles collection and all files from every sourceJars. Name of Artifact generated based on - * outputJar. - * - * @param outputJar name of output Jar artifact. - * @param outputSourceJar name of output source Jar artifact, or {@code null}. If unset, defaults - * to base name of the output jar with the suffix {@code -src.jar}. - * @return generated artifact (can also be empty) - */ - Artifact packSourceFiles( - StarlarkActionFactory actions, - Artifact outputJar, - Artifact outputSourceJar, - List sourceFiles, - List sourceJars, - JavaToolchainProvider javaToolchain, - String execGroup) - throws EvalException { - if (outputJar == null && outputSourceJar == null) { - throw Starlark.errorf( - "pack_sources requires at least one of the parameters output_jar or output_source_jar"); - } - // If we only have one source jar, return it directly to avoid action creation - if (sourceFiles.isEmpty() && sourceJars.size() == 1 && outputSourceJar == null) { - return sourceJars.get(0); - } - ActionRegistry actionRegistry = actions.asActionRegistry(actions); - if (outputSourceJar == null) { - outputSourceJar = getDerivedSourceJar(actions.getActionConstructionContext(), outputJar); - } - SingleJarActionBuilder.createSourceJarAction( - actionRegistry, - actions.getActionConstructionContext(), - javaToolchain.getJavaSemantics(), - NestedSetBuilder.wrap(Order.STABLE_ORDER, sourceFiles), - NestedSetBuilder.wrap(Order.STABLE_ORDER, sourceJars), - outputSourceJar, - javaToolchain, - execGroup); - return outputSourceJar; - } - private JavaSourceJarsProvider createJavaSourceJarsProvider( Iterable sourceJars, Iterable transitiveDeps) { NestedSetBuilder transitiveSourceJars = NestedSetBuilder.stableOrder(); diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java index 1b9741db719bd3..b83c34f294af16 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaStarlarkCommon.java @@ -14,7 +14,6 @@ package com.google.devtools.build.lib.rules.java; import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.google.devtools.build.lib.packages.ExecGroup.DEFAULT_EXEC_GROUP_NAME; import com.google.common.base.Predicates; import com.google.common.collect.ImmutableList; @@ -203,35 +202,6 @@ public JavaInfo createJavaCompileAction( thread); } - private String getExecGroup(boolean useAutoExecGroups) { - if (useAutoExecGroups) { - return javaSemantics.getJavaToolchainType(); - } else { - return DEFAULT_EXEC_GROUP_NAME; - } - } - - @Override - public Artifact packSources( - StarlarkActionFactory actions, - Object outputJar, - Object outputSourceJar, - Sequence sourceFiles, // expected. - Sequence sourceJars, // expected. - JavaToolchainProvider javaToolchain, - Object hostJavabase) - throws EvalException { - return JavaInfoBuildHelper.getInstance() - .packSourceFiles( - actions, - outputJar instanceof Artifact ? (Artifact) outputJar : null, - outputSourceJar instanceof Artifact ? (Artifact) outputSourceJar : null, - Sequence.cast(sourceFiles, Artifact.class, "sources"), - Sequence.cast(sourceJars, Artifact.class, "source_jars"), - javaToolchain, - getExecGroup(actions.getRuleContext().useAutoExecGroups())); - } - @Override // TODO(b/78512644): migrate callers to passing explicit javacopts or using custom toolchains, and // delete diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java index 41ca4264c8f154..e6825686d81f93 100644 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/java/JavaCommonApi.java @@ -29,7 +29,6 @@ import com.google.devtools.build.lib.starlarkbuildapi.core.TransitiveInfoCollectionApi; import com.google.devtools.build.lib.starlarkbuildapi.cpp.CcInfoApi; import com.google.devtools.build.lib.starlarkbuildapi.platform.ConstraintValueInfoApi; -import javax.annotation.Nullable; import net.starlark.java.annot.Param; import net.starlark.java.annot.ParamType; import net.starlark.java.annot.StarlarkBuiltin; @@ -318,83 +317,6 @@ JavaInfoT createJavaCompileAction( StarlarkThread thread) throws EvalException, InterruptedException, RuleErrorException; - @StarlarkMethod( - name = "pack_sources", - doc = - "Packs sources and source jars into a single source jar file. " - + "The return value is typically passed to" - + "

" - + "JavaInfo#source_jar

." - + "At least one of parameters output_jar or output_source_jar is required.", - parameters = { - @Param(name = "actions", named = true, doc = "ctx.actions"), - @Param( - name = "output_jar", - positional = false, - named = true, - allowedTypes = { - @ParamType(type = FileApi.class), - @ParamType(type = NoneType.class), - }, - defaultValue = "None", - doc = - "Deprecated: The output jar of the rule. Used to name the resulting source jar. " - + "The parameter sets output_source_jar parameter to `{output_jar}-src.jar`." - + "Use output_source_jar parameter directly instead.", - disableWithFlag = BuildLanguageOptions.INCOMPATIBLE_JAVA_COMMON_PARAMETERS, - valueWhenDisabled = "None"), - @Param( - name = "output_source_jar", - positional = false, - named = true, - allowedTypes = { - @ParamType(type = FileApi.class), - @ParamType(type = NoneType.class), - }, - defaultValue = "None", - doc = "The output source jar."), - @Param( - name = "sources", - positional = false, - named = true, - allowedTypes = {@ParamType(type = Sequence.class, generic1 = FileApi.class)}, - defaultValue = "[]", - doc = "A list of Java source files to be packed into the source jar."), - @Param( - name = "source_jars", - positional = false, - named = true, - allowedTypes = {@ParamType(type = Sequence.class, generic1 = FileApi.class)}, - defaultValue = "[]", - doc = "A list of source jars to be packed into the source jar."), - @Param( - name = "java_toolchain", - positional = false, - named = true, - doc = "A JavaToolchainInfo to used to find the ijar tool."), - @Param( - name = "host_javabase", - positional = false, - named = true, - doc = - "Deprecated: You can drop this parameter (host_javabase is provided with " - + "java_toolchain)", - defaultValue = "None", - disableWithFlag = BuildLanguageOptions.INCOMPATIBLE_JAVA_COMMON_PARAMETERS, - valueWhenDisabled = "None"), - }, - allowReturnNones = true) - @Nullable - FileApi packSources( - StarlarkActionFactoryT actions, - Object outputJar, - Object outputSourceJar, - Sequence sourceFiles, // expected. - Sequence sourceJars, // expected. - JavaToolchainT javaToolchain, - Object hostJavabase) - throws EvalException; - @StarlarkMethod( name = "default_javac_opts", // This function is experimental for now. diff --git a/src/main/starlark/builtins_bzl/common/java/java_common.bzl b/src/main/starlark/builtins_bzl/common/java/java_common.bzl index 848f25c59e1bc4..e8d8dc962450bc 100644 --- a/src/main/starlark/builtins_bzl/common/java/java_common.bzl +++ b/src/main/starlark/builtins_bzl/common/java/java_common.bzl @@ -24,6 +24,7 @@ load(":common/java/java_common_internal_for_builtins.bzl", "compile", "merge", " load(":common/java/java_plugin_info.bzl", "JavaPluginInfo") load(":common/java/java_semantics.bzl", "semantics") load(":common/paths.bzl", "paths") +load(":common/java/java_helper.bzl", "helper") _java_common_internal = _builtins.internal.java_common_internal_do_not_use @@ -123,13 +124,12 @@ def _stamp_jar(actions, jar, java_toolchain, target_label): def _pack_sources( actions, java_toolchain, - output_source_jar = None, + output_source_jar, sources = [], source_jars = []): """Packs sources and source jars into a single source jar file. - The return value is typically passed to `JavaInfo.source_jar`. At least one of parameters - output_jar or output_source_jar is required. + The return value is typically passed to `JavaInfo.source_jar`. Args: actions: (actions) ctx.actions @@ -141,12 +141,14 @@ def _pack_sources( Returns: (File) The output artifact """ - return _java_common_internal.pack_sources( - actions = actions, - java_toolchain = java_toolchain, - sources = sources, - source_jars = source_jars, - output_source_jar = output_source_jar, + return helper.create_single_jar( + actions, + toolchain = java_toolchain, + output = output_source_jar, + sources = depset(source_jars), + resources = depset(sources), + progress_message = "Building source jar %{output}", + mnemonic = "JavaSourceJar", ) def _default_javac_opts(java_toolchain): diff --git a/src/test/java/com/google/devtools/build/lib/analysis/AutoExecGroupsTest.java b/src/test/java/com/google/devtools/build/lib/analysis/AutoExecGroupsTest.java index fd186783b4f5fd..0b9286a525a942 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/AutoExecGroupsTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/AutoExecGroupsTest.java @@ -26,7 +26,6 @@ import com.google.devtools.build.lib.actions.ActionAnalysisMetadata; import com.google.devtools.build.lib.analysis.actions.LazyWritePathsFileAction; import com.google.devtools.build.lib.analysis.actions.ParameterFileWriteAction; -import com.google.devtools.build.lib.analysis.actions.SpawnAction; import com.google.devtools.build.lib.analysis.config.ToolchainTypeRequirement; import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget; import com.google.devtools.build.lib.analysis.platform.ToolchainInfo; @@ -1532,10 +1531,12 @@ public void javaCommonPackSources_automaticExecGroupsEnabled_sourceActionExecute "custom_rule(name = 'custom_rule_name')"); useConfiguration("--incompatible_auto_exec_groups"); - ImmutableList actions = getActions("//test:custom_rule_name", SpawnAction.class); + ImmutableList actions = + getActions("//test:custom_rule_name").stream() + .filter(action -> action.getMnemonic().equals("JavaSourceJar")) + .collect(toImmutableList()); assertThat(actions).hasSize(1); - assertThat(actions.get(0).getMnemonic()).isEqualTo("JavaSourceJar"); assertThat(actions.get(0).getOwner().getExecutionPlatform().label()) .isEqualTo(Label.parseCanonical("//platforms:platform_1")); } From 95c15c85b17df8e6976e9a17eb3c7fa5f29e3af5 Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 5 Jul 2023 14:52:23 -0700 Subject: [PATCH 49/51] Automated rollback of commit 990d97e576d4ec7d0c45f3efa5732171492d50b1. *** Reason for rollback *** This is a roll forward CL which addressed the issue reported in b/289354550#comment9. I added another `LabelInterner#enabled()` method to check whether `globalPool` was set or not. If not, then `SkyframeExecutor` should not remove any labels from weak interner when PAKCAGE node is done. A unit test is also added to verify the condition when `LabelInterner#enable()` returns false. *** Original change description *** Automated rollback of commit 0bda661e589ded1caad9edd58c9bebc3f647e41d. *** Reason for rollback *** b/289354550 *** Original change description *** Clean up Label Interner flag and relevant unused code PiperOrigin-RevId: 545793223 Change-Id: I4254c6ed910e31c2702a26ddec9c334470f7e1bf --- .../devtools/build/lib/cmdline/Label.java | 16 +++---- .../build/lib/skyframe/SkyframeExecutor.java | 2 +- .../com/google/devtools/build/skyframe/BUILD | 1 - .../build/skyframe/InMemoryGraphImpl.java | 11 +---- .../skyframe/UsePooledLabelInterningFlag.java | 44 ------------------- .../google/devtools/build/lib/cmdline/BUILD | 1 - .../cmdline/LabelInternerIntegrationTest.java | 44 +++++++++++++++++++ .../com/google/devtools/build/skyframe/BUILD | 1 - 8 files changed, 53 insertions(+), 67 deletions(-) delete mode 100644 src/main/java/com/google/devtools/build/skyframe/UsePooledLabelInterningFlag.java diff --git a/src/main/java/com/google/devtools/build/lib/cmdline/Label.java b/src/main/java/com/google/devtools/build/lib/cmdline/Label.java index 25e432e4b9346d..9aeb87277ace2e 100644 --- a/src/main/java/com/google/devtools/build/lib/cmdline/Label.java +++ b/src/main/java/com/google/devtools/build/lib/cmdline/Label.java @@ -19,7 +19,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ComparisonChain; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Interner; import com.google.common.util.concurrent.Striped; import com.google.devtools.build.docgen.annot.DocCategory; import com.google.devtools.build.lib.actions.CommandLineItem; @@ -34,7 +33,6 @@ import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.skyframe.SkyFunctionName; import com.google.devtools.build.skyframe.SkyKey; -import com.google.devtools.build.skyframe.UsePooledLabelInterningFlag; import java.util.Arrays; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; @@ -86,16 +84,10 @@ public final class Label implements Comparable