diff --git a/.bazelci/postsubmit.yml b/.bazelci/postsubmit.yml index f0a1eef5a12a4a..047fd07308fc4d 100644 --- a/.bazelci/postsubmit.yml +++ b/.bazelci/postsubmit.yml @@ -87,6 +87,7 @@ tasks: - "--sandbox_writable_path=$HOME/bazeltest" - "--test_env=TEST_INSTALL_BASE=$HOME/bazeltest/install_base" - "--test_env=TEST_REPOSITORY_HOME=$OUTPUT_BASE/external" + - "--test_tag_filters=-no_1804" # Configure and enable tests that require access to the network. - "--test_env=REMOTE_NETWORK_ADDRESS=bazel.build:80" test_targets: @@ -138,6 +139,7 @@ tasks: - "--sandbox_writable_path=$HOME/bazeltest" - "--test_env=TEST_INSTALL_BASE=$HOME/bazeltest/install_base" - "--test_env=TEST_REPOSITORY_HOME=$OUTPUT_BASE/external" + - "--test_tag_filters=-no_1804" test_targets: - "//src/test/shell/bazel:cc_integration_test" include_json_profile: @@ -311,8 +313,6 @@ tasks: - "-//src/test/java/com/google/devtools/build/lib/skyframe/rewinding:RewindingTest" - "-//src/test/java/com/google/devtools/build/lib/buildtool:MiscAnalysisTest" - "-//src/test/java/com/google/devtools/build/lib/rules/objc:ObjcRulesTests" - # https://github.com/bazelbuild/bazel/issues/16975 - - "-//scripts/docs:rewriter_test" # https://github.com/bazelbuild/bazel/issues/17007 - "-//src/test/java/com/google/devtools/build/lib/platform:SystemMemoryPressureEventTest" include_json_profile: @@ -340,6 +340,7 @@ tasks: - "--host_copt=-w" - "--test_tag_filters=-no_windows,-slow" - "--test_env=JAVA_HOME" + - "--test_env=BAZEL_VC" - "--test_env=TEST_INSTALL_BASE=$HOME/bazeltest_install_base" - "--test_env=TEST_REPOSITORY_HOME=C:/b/bazeltest_external" test_targets: @@ -403,6 +404,7 @@ tasks: - "--experimental_remote_download_outputs=minimal" - "--experimental_inmemory_jdeps_files" - "--experimental_inmemory_dotd_files" + - "--test_tag_filters=-no_1804" test_targets: - "//scripts/..." - "//src/java_tools/..." diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 0e8010fb16316d..fe30ced7d83907 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -50,6 +50,7 @@ tasks: - "//third_party/ijar/..." - "//tools/android/..." - "//tools/aquery_differ/..." + - "//tools/compliance/..." - "//tools/python/..." # Re-enable once fixed: https://github.com/bazelbuild/bazel/issues/8162 - "-//src/java_tools/buildjar/..." @@ -91,6 +92,7 @@ tasks: - "--sandbox_writable_path=$HOME/bazeltest" - "--test_env=TEST_INSTALL_BASE=$HOME/bazeltest/install_base" - "--test_env=TEST_REPOSITORY_HOME=$OUTPUT_BASE/external" + - "--test_tag_filters=-no_1804" # Configure and enable tests that require access to the network. - "--test_env=REMOTE_NETWORK_ADDRESS=bazel.build:80" test_targets: @@ -142,6 +144,7 @@ tasks: - "--sandbox_writable_path=$HOME/bazeltest" - "--test_env=TEST_INSTALL_BASE=$HOME/bazeltest/install_base" - "--test_env=TEST_REPOSITORY_HOME=$OUTPUT_BASE/external" + - "--test_tag_filters=-no_1804" test_targets: - "//src/test/shell/bazel:cc_integration_test" include_json_profile: @@ -322,8 +325,6 @@ tasks: - "-//src/test/java/com/google/devtools/build/lib/skyframe/rewinding:RewindingTest" - "-//src/test/java/com/google/devtools/build/lib/buildtool:MiscAnalysisTest" - "-//src/test/java/com/google/devtools/build/lib/rules/objc:ObjcRulesTests" - # https://github.com/bazelbuild/bazel/issues/16975 - - "-//scripts/docs:rewriter_test" # https://github.com/bazelbuild/bazel/issues/17007 - "-//src/test/java/com/google/devtools/build/lib/platform:SystemMemoryPressureEventTest" # Disable the most time-consuming tests on macOS arm64 platform in presubmit. @@ -406,6 +407,7 @@ tasks: - "--host_copt=-w" - "--test_tag_filters=-no_windows,-slow" - "--test_env=JAVA_HOME" + - "--test_env=BAZEL_VC" - "--test_env=TEST_INSTALL_BASE=$HOME/bazeltest_install_base" - "--test_env=TEST_REPOSITORY_HOME=C:/b/bazeltest_external" test_targets: @@ -471,6 +473,7 @@ tasks: - "--experimental_remote_cache_async" - "--experimental_remote_merkle_tree_cache" - "--remote_download_minimal" + - "--test_tag_filters=-no_1804" test_targets: - "//scripts/..." - "//src/java_tools/..." diff --git a/.github/ISSUE_TEMPLATE/mirror_request.yml b/.github/ISSUE_TEMPLATE/mirror_request.yml index 2f315bad771969..5521cf18d9f937 100644 --- a/.github/ISSUE_TEMPLATE/mirror_request.yml +++ b/.github/ISSUE_TEMPLATE/mirror_request.yml @@ -9,12 +9,11 @@ body: - type: markdown attributes: value: > - **Attention:** if the archive you're trying to mirror is a GitHub release archive, - please use URLs of the form `https://github.com/$USER/$REPO/archive/refs/tags/$TAG`, - instead of the form without the `refs/tags/` part. The latter is *not* guaranteed to - have a stable hash (see - https://github.com/bazel-contrib/SIG-rules-authors/issues/11#issuecomment-1029861300 - for more details). + **Attention:** if the archive you're trying to mirror is from GitHub, + please use URLs in the form of `https://github.com/$USER/$REPO/releases/download/...` if available. + If you are the project maintainer, you should create and upload such an release archive. + GitHub doesn't guarantee a stable checksum of source archives in the form of `https://github.com///archive/...`, which are generated on demand. + Check [GitHub Archive Checksum Outage](https://blog.bazel.build/2023/02/15/github-archive-checksum.html) for more details. - type: textarea id: urls attributes: diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 3109f7f3edaf5b..5fcb5010d3c568 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -33,11 +33,11 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-issue-stale: 430 days-before-pr-stale: 430 - days-before-issue-close: 14 - days-before-pr-close: 14 + days-before-issue-close: 90 + days-before-pr-close: 90 stale-issue-message: > Thank you for contributing to the Bazel repository! - This issue has been marked as stale since it has not had any activity in the last 1+ years. It will be closed in the next 14 + This issue has been marked as stale since it has not had any activity in the last 1+ years. It will be closed in the next 90 days unless any other activity occurs or one of the following labels is added: "not stale", "awaiting-bazeler". Please reach out to the triage team (`@bazelbuild/triage`) if you think this issue is still relevant or you are interested in getting the issue resolved. @@ -47,17 +47,17 @@ jobs: stale-pr-message: > Thank you for contributing to the Bazel repository! This pull request has been marked as stale since it has not had any activity in the last 1+ years. It will be closed in the next - 14 days unless any other activity occurs or one of the following labels is added: "not stale", "awaiting-review", "awaiting-PR-merge". + 90 days unless any other activity occurs or one of the following labels is added: "not stale", "awaiting-review", "awaiting-PR-merge". Please reach out to the triage team (`@bazelbuild/triage`) if you think this PR is still relevant or you are interested in getting the PR merged. close-pr-message: > This pull request has been automatically closed due to inactivity. If you're still interested in pursuing this, please reach out to the triage team (`@bazelbuild/triage`). Thanks! stale-issue-label: 'stale' - exempt-issue-labels: 'not stale,awaiting-bazeler,untriaged,P0,P1' + exempt-issue-labels: 'not stale,awaiting-bazeler,untriaged,P0,P1,P2,good first issue,help wanted' close-issue-reason: "not_planned" stale-pr-label: 'stale' - exempt-pr-labels: 'not stale,awaiting-review,awaiting-PR-merge,P0,P1' + exempt-pr-labels: 'not stale,awaiting-review,awaiting-PR-merge,P0,P1,P2' exempt-draft-pr: true operations-per-run: 500 ascending: true diff --git a/BUILD b/BUILD index d00a05d228d3a8..1e051df359cb69 100644 --- a/BUILD +++ b/BUILD @@ -181,6 +181,11 @@ pkg_tar( visibility = ["//:__subpackages__"], ) +exports_files( + ["maven_install.json"], + visibility = ["//tools/compliance:__pkg__"], +) + py_binary( name = "combine_distfiles", srcs = ["combine_distfiles.py"], diff --git a/WORKSPACE b/WORKSPACE index c7ae1fa85dae7b..dff2ce748594f9 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -473,14 +473,13 @@ maven_install( "io.netty:netty-transport-native-unix-common:jar:linux-x86_64:4.1.93.Final", "io.netty:netty-transport-native-unix-common:jar:osx-aarch_64:4.1.93.Final", "io.netty:netty-transport-native-unix-common:jar:osx-x86_64:4.1.93.Final", - "io.netty:netty-transport-sctp:4.1.93.Final", "io.netty:netty-transport:4.1.93.Final", "io.reactivex.rxjava3:rxjava:3.1.2", "javax.activation:javax.activation-api:1.2.0", "javax.annotation:javax.annotation-api:1.3.2", "javax.inject:javax.inject:1", - "net.bytebuddy:byte-buddy-agent:1.11.13", - "net.bytebuddy:byte-buddy:1.11.13", + "net.bytebuddy:byte-buddy-agent:1.14.5", + "net.bytebuddy:byte-buddy:1.14.5", "org.apache.commons:commons-compress:1.19", "org.apache.commons:commons-pool2:2.8.0", "org.apache.tomcat:tomcat-annotations-api:8.0.5", @@ -559,7 +558,8 @@ maven_install( "com.google.protobuf:protobuf-java", "com.google.protobuf:protobuf-javalite", ], - fail_if_repin_required = False, + # Don't forget to change this to back to True before submitting your change. + fail_if_repin_required = True, maven_install_json = "//:maven_install.json", repositories = [ "https://repo1.maven.org/maven2", @@ -583,6 +583,7 @@ maven_install( "com.android.tools:common:30.1.3", "com.android.tools:repository:30.1.3", ], + # Don't forget to change this to back to True before submitting your change. fail_if_repin_required = True, maven_install_json = "//src/tools/android:maven_android_install.json", repositories = [ diff --git a/maven_install.json b/maven_install.json index 222bb478867ac1..6abdccf2de2a85 100644 --- a/maven_install.json +++ b/maven_install.json @@ -1,7 +1,7 @@ { "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": "THERE_IS_NO_DATA_ONLY_ZUUL", - "__INPUT_ARTIFACTS_HASH": -482172833, - "__RESOLVED_ARTIFACTS_HASH": -552166642, + "__INPUT_ARTIFACTS_HASH": 307745718, + "__RESOLVED_ARTIFACTS_HASH": -1204291120, "artifacts": { "aopalliance:aopalliance": { "shasums": { @@ -475,12 +475,6 @@ }, "version": "4.1.93.Final" }, - "io.netty:netty-transport-sctp": { - "shasums": { - "jar": "9021704f3ad2758d3d2728d335fcd16df8390bbcd9cfb64ee24e2024d4cd5a0f" - }, - "version": "4.1.93.Final" - }, "io.opencensus:opencensus-api": { "shasums": { "jar": "f1474d47f4b6b001558ad27b952e35eda5cc7146788877fc52938c6eba24b382" @@ -537,15 +531,15 @@ }, "net.bytebuddy:byte-buddy": { "shasums": { - "jar": "e29fa75b903432ac64d05c18c19d0e3b9026e74abda52bbd6f9065e55f4a29f5" + "jar": "e99761a526df0fefbbd3fe14436b0f953000cdfa5151dc63c0b18d37d9c46f1c" }, - "version": "1.11.13" + "version": "1.14.5" }, "net.bytebuddy:byte-buddy-agent": { "shasums": { - "jar": "49b43b0d10f8bb1d800d56137bdf0f44628412ebe1fbd804e45f363d495860fa" + "jar": "55f19862b870f5d85890ba5386b1b45e9bbc88d5fe1f819abe0c788b4929fa6b" }, - "version": "1.11.13" + "version": "1.14.5" }, "org.apache.commons:commons-compress": { "shasums": { @@ -603,15 +597,15 @@ }, "org.mockito:mockito-core": { "shasums": { - "jar": "75d4b14ba7aef836e92ba7b2d53ca7d6b215dd7db5625afbc39252f1358835fe" + "jar": "b1689b06617ea01fd777bfaedbdde512faf083d639a049f79b388d5a4e96d2e5" }, - "version": "3.12.4" + "version": "5.4.0" }, "org.objenesis:objenesis": { "shasums": { - "jar": "03d960bd5aef03c653eb000413ada15eb77cdd2b8e4448886edf5692805e35f3" + "jar": "02dfd0b0439a5591e35b708ed2f5474eb0948f53abf74637e959b8e4ef69bfeb" }, - "version": "3.2" + "version": "3.3" }, "org.ow2.asm:asm": { "shasums": { @@ -1062,12 +1056,6 @@ "io.netty:netty-common", "io.netty:netty-transport" ], - "io.netty:netty-transport-sctp": [ - "io.netty:netty-buffer", - "io.netty:netty-codec", - "io.netty:netty-common", - "io.netty:netty-transport" - ], "io.opencensus:opencensus-api": [ "io.grpc:grpc-context" ], @@ -1652,12 +1640,6 @@ "io.netty:netty-transport-native-unix-common:jar:osx-x86_64": [ "io.netty.channel.unix" ], - "io.netty:netty-transport-sctp": [ - "io.netty.channel.sctp", - "io.netty.channel.sctp.nio", - "io.netty.channel.sctp.oio", - "io.netty.handler.codec.sctp" - ], "io.opencensus:opencensus-api": [ "io.opencensus.common", "io.opencensus.internal", @@ -1802,11 +1784,13 @@ "net.bytebuddy.pool", "net.bytebuddy.utility", "net.bytebuddy.utility.dispatcher", + "net.bytebuddy.utility.nullability", "net.bytebuddy.utility.privilege", "net.bytebuddy.utility.visitor" ], "net.bytebuddy:byte-buddy-agent": [ - "net.bytebuddy.agent" + "net.bytebuddy.agent", + "net.bytebuddy.agent.utility.nullability" ], "org.apache.commons:commons-compress": [ "org.apache.commons.compress", @@ -2005,7 +1989,6 @@ "org.mockito.internal.invocation.finder", "org.mockito.internal.invocation.mockref", "org.mockito.internal.junit", - "org.mockito.internal.junit.util", "org.mockito.internal.listeners", "org.mockito.internal.matchers", "org.mockito.internal.matchers.apachecommons", @@ -2033,7 +2016,6 @@ "org.mockito.mock", "org.mockito.plugins", "org.mockito.quality", - "org.mockito.runners", "org.mockito.session", "org.mockito.stubbing", "org.mockito.verification" @@ -2181,7 +2163,6 @@ "io.netty:netty-transport-native-unix-common:jar:linux-x86_64", "io.netty:netty-transport-native-unix-common:jar:osx-aarch_64", "io.netty:netty-transport-native-unix-common:jar:osx-x86_64", - "io.netty:netty-transport-sctp", "io.opencensus:opencensus-api", "io.opencensus:opencensus-contrib-http-util", "io.perfmark:perfmark-api", diff --git a/site/en/_book.yaml b/site/en/_book.yaml index f9528a35e187ea..1715cc2d31cd41 100644 --- a/site/en/_book.yaml +++ b/site/en/_book.yaml @@ -130,6 +130,8 @@ upper_tabs: path: /external/registry - title: Module extensions path: /external/extension + - title: Bzlmod migration guide + path: /external/migration - title: Advanced topics path: /external/advanced - heading: Querying your build diff --git a/site/en/contribute/support.md b/site/en/contribute/support.md deleted file mode 100644 index 32cb7e37ff3544..00000000000000 --- a/site/en/contribute/support.md +++ /dev/null @@ -1,21 +0,0 @@ -Project: /_project.yaml -Book: /_book.yaml - -# Support Policy - -{% include "_buttons.html" %} - -The Bazel team generally avoids making backwards-incompatible changes. However, -these changes are sometimes necessary to fix bugs, make improvements (such as -improving performance or usability) to the system, or to lock down APIs that -are known to be brittle. - -Major changes are announced in advance on the -[bazel-discuss](https://groups.google.com/forum/#!forum/bazel-discuss){: .external} mailing -list. Both undocumented features (attributes, rules, "Make" variables, and -flags) and documented features that are marked *experimental* are subject to -change at any time without prior notice. - -Report any bugs or regressions you find on -[GitHub](https://github.com/bazelbuild/bazel/issues){: .external}. The repository maintainers -make an effort to triage reported issues within 2 business days. diff --git a/site/en/docs/_index.yaml b/site/en/docs/_index.yaml index 200e6f5b6922b2..177e2b0862e945 100644 --- a/site/en/docs/_index.yaml +++ b/site/en/docs/_index.yaml @@ -49,7 +49,7 @@ landing_page: Learn about Bazel's release model, latest releases, and compatibility policies. items_across: 4 items: - - heading: Release policy + - heading: Release model path: /release/ - heading: Backward compatibility path: /release/backward-compatibility diff --git a/site/en/external/migration.md b/site/en/external/migration.md new file mode 100644 index 00000000000000..de9a5ba65155ee --- /dev/null +++ b/site/en/external/migration.md @@ -0,0 +1,803 @@ +Project: /_project.yaml +Book: /_book.yaml +keywords: bzlmod + +# Bzlmod Migration Guide + +{% include "_buttons.html" %} + +Due to the [shortcomings of +WORKSPACE](/external/overview#workspace-shortcomings), Bzlmod is going to +replace the legacy WORKSPACE system in future Bazel releases. This guide helps +you migrate your project to Bzlmod and drop WORKSPACE for fetching external +dependencies. + +## WORKSPACE vs Bzlmod {:#workspace-vs-bzlmod} + +Bazel's WORKSPACE and Bzlmod offer similar features with different syntax. This +section explains how to migrate from specific WORKSPACE functionalities to +Bzlmod. + +### Define the root of a Bazel workspace {:#define-root} + +The WORKSPACE file marks the source root of a Bazel project, this responsibility +is replaced by MODULE.bazel in Bazel version 6.3 and later. With Bazel version +prior to 6.3, there should still be a `WORKSPACE` or `WORKSPACE.bazel` file at +your workspace root, maybe with comments like: + +* **WORKSPACE** + + ```python + # This file marks the root of the Bazel workspace. + # See MODULE.bazel for external dependencies setup. + ``` + +### Specify repository name for your workspace {:#specify-repo-name} + +* **WORKSPACE** + + The [`workspace`](/rules/lib/globals/workspace#workspace) function is used + to specify a repository name for your workspace. This allows a target + `//foo:bar` in the workspace to be referenced as `@//foo:bar`. If not specified, the default repository name for your + workspace is `__main__`. + + ```python + ## WORKSPACE + workspace(name = "com_foo_bar") + ``` + +* **Bzlmod** + + It's recommended to reference targets in the same workspace with the + `//foo:bar` syntax without `@`. But if you do need the old syntax + , you can use the module name specified by the + [`module`](/rules/lib/globals/module#module) function as the repository + name. If the module name is different from the needed repository name, you + can use `repo_name` attribute of the + [`module`](/rules/lib/globals/module#module) function to override the + repository name. + + ```python + ## MODULE.bazel + module( + name = "bar", + repo_name = "com_foo_bar", + ) + ``` + +### Fetch external dependencies as Bazel modules {:#fetch-bazel-modules} + +If your dependency is a Bazel project, you should be able to depend on it as a +Bazel module when it also adopts Bzlmod. + +* **WORKSPACE** + + With WORKSPACE, it's common to use the + [`http_archive`](/rules/lib/repo/http#http_archive) or + [`git_repository`](/rules/lib/repo/git#git_repository) repository rules to + download the sources of the Bazel project. + + ```python + ## WORKSPACE + load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + + http_archive( + name = "bazel_skylib", + urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/1.4.2/bazel-skylib-1.4.2.tar.gz"], + sha256 = "66ffd9315665bfaafc96b52278f57c7e2dd09f5ede279ea6d39b2be471e7e3aa", + ) + load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") + bazel_skylib_workspace() + + http_archive( + name = "rules_java", + urls = ["https://github.com/bazelbuild/rules_java/releases/download/6.1.1/rules_java-6.1.1.tar.gz"], + sha256 = "76402a50ae6859d50bd7aed8c1b8ef09dae5c1035bb3ca7d276f7f3ce659818a", + ) + load("@rules_java//java:repositories.bzl", "rules_java_dependencies", "rules_java_toolchains") + rules_java_dependencies() + rules_java_toolchains() + ``` + + As you can see, it's a common pattern that users need to load transitive + dependencies from a macro of the dependency. Assume both `bazel_skylib` and + `rules_java` depends on `platoform`, the exact version of the `platform` + dependency is determined by the order of the macros. + +* **Bzlmod** + + With Bzlmod, as long as the your dependency is available in [Bazel Central + Registry](https://registry.bazel.build) or your custom [Bazel + registry](/external/registry), you can simply depend on it with a + [`bazel_dep`](/rules/lib/globals/module#bazel_dep) directive. + + ```python + ## MODULE.bazel + bazel_dep(name = "bazel_skylib", version = "1.4.2") + bazel_dep(name = "rules_java", version = "6.1.1") + ``` + + Bzlmod resolves Bazel module dependencies transitively using the + [MVS](https://research.swtch.com/vgo-mvs) algorithm. Therefore, the maximal + required version of `platform` is selected automatically. + +### Override a dependency as a Bazel module{:#override-modules} + +As the root module, you can override Bazel module dependencies in different +ways. + +Please read the [overrides](/external/module#overrides) section for more +information. + +You can find some example usages in the +[examples][override-examples] +repository. + +[override-examples]: https://github.com/bazelbuild/examples/blob/main/bzlmod/02-override_bazel_module + +### Fetch external dependencies with module extensions{:#fetch-deps-module-extensions} + +If your dependency is not a Bazel project or not yet available in any Bazel +registry, you can introduce it using [module extensions](/external/extension). + +* **WORKSPACE** + + Download a file using the [`http_file`](/rules/lib/repo/http#http_file) + repository rule. + + ```python + ## WORKSPACE + load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") + + http_file( + name = "data_file", + url = "http://example.com/file", + sha256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ) + ``` + +* **Bzlmod** + + With Bzlmod, you have to move the definition into a `.bzl` file, which also + lets you share the definition between WORKSPACE and Bzlmod during the + migration period. + + ```python + ## repositories.bzl + load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") + def my_data_dependency(): + http_file( + name = "data_file", + url = "http://example.com/file", + sha256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ) + ``` + + Implement a module extension to load the dependencies macro. You can define + it in the same `.bzl` file of the macro, but to keep compatibility with + older Bazel versions, it's better to define it in a separate `.bzl` file. + + ```python + ## extensions.bzl + load("//:repositories.bzl", "my_data_dependency") + def _non_module_dependencies_impl(_ctx): + my_data_dependency() + + non_module_dependencies = module_extension( + implementation = _non_module_dependencies_impl, + ) + ``` + + To make the repository visible to the root project, you should declare the + usages of the module extension and the repository in the MODULE.bazel file. + + ```python + ## MODULE.bazel + non_module_dependencies = use_extension("//:extensions.bzl", "non_module_dependencies") + use_repo(non_module_dependencies, "data_file") + ``` + +### Resolve conflict external dependencies with module extension {:#conflict-deps-module-extension} + +A project can provide a macro that introduces external repositories based on +inputs from its callers. But what if there are multiple callers in the +dependency graph and they cause a conflict? + +Assume the project `foo` provides the following macro which takes `version` as +an argument. + +```python +## repositories.bzl in foo {:#repositories.bzl-foo} +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") +def data_deps(version = "1.0"): + http_file( + name = "data_file", + url = "http://example.com/file-%s" % version, + # Omitting the "sha256" attribute for simplicity + ) +``` + +* **WORKSPACE** + + With WORKSPACE, you can load the macro from `@foo` and specify the version + of the data dependency you need. Assume you have another dependency `@bar`, + which also depends on `@foo` but requires a different version of the data + dependency. + + ```python + ## WORKSPACE + + # Introduce @foo and @bar. + ... + + load("@foo//:repositories.bzl", "data_deps") + data_deps(version = "2.0") + + load("@bar//:repositories.bzl", "bar_deps") + bar_deps() # -> which calls data_deps(version = "3.0") + ``` + + In this case, the end user must carefully adjust the order of macros in the + WORKSPACE to get the version they need. This is one of the biggest pain + points with WORKSPACE since it doesn't really provide a sensible way to + resolve dependencies. + +* **Bzlmod** + + With Bzlmod, the author of project `foo` can use module extension to resolve + conflicts. For example, let's assume it makes sense to always select the + maximal required version of the data dependency among all Bazel modules. + + ```python + ## extensions.bzl in foo + load("//:repositories.bzl", "data_deps") + + data = tag_class(attrs={"version": attr.string()}) + + def _data_deps_extension_impl(module_ctx): + # Select the maximal required version in the dependency graph. + version = "1.0" + for mod in module_ctx.modules: + for data in mod.tags.data: + version = max(version, data.version) + data_deps(version) + + data_deps_extension = module_extension( + implementation = _data_deps_extension_impl, + tag_classes = {"data": data}, + ) + ``` + + ```python + ## MODULE.bazel in bar + bazel_dep(name = "foo", version = "1.0") + + foo_data_deps = use_extension("@foo//:extensions.bzl", "data_deps_extension") + foo_data_deps.data(version = "3.0") + use_repo(foo_data_deps, "data_file") + ``` + + ```python + ## MODULE.bazel in root module + bazel_dep(name = "foo", version = "1.0") + bazel_dep(name = "bar", version = "1.0") + + foo_data_deps = use_extension("@foo//:extensions.bzl", "data_deps_extension") + foo_data_deps.data(version = "2.0") + use_repo(foo_data_deps, "data_file") + ``` + + In this case, the root module requires data version `2.0`, while its + dependency `bar` requires `3.0`. The module extension in `foo` can correctly + resolve this conflict and automatically select version `3.0` for the data + dependency. + +### Integrate third party package manager {:#integrate-package-manager} + +Following the last section, since module extension provides a way to collect +information from the dependency graph, perform custom logic to resolve +dependencies and call repository rules to introduce external repositories, this +provides a great way for rules authors to enhance the rulesets that integrate +package managers for specific languages. + +Please read the [module extensions](/external/extension) page to learn more +about how to use module extensions. + +Here is a list of the rulesets that already adopted Bzlmod to fetch dependencies +from different package managers: + +- [rules_jvm_external](https://github.com/bazelbuild/rules_jvm_external/blob/master/docs/bzlmod.md) +- [rules_go](https://github.com/bazelbuild/rules_go/blob/master/docs/go/core/bzlmod.md) +- [rules_python](https://github.com/bazelbuild/rules_python/blob/main/BZLMOD_SUPPORT.md) + +A minimal example that integrates a pseudo package manager is available at the +[examples][pkg-mgr-example] +repository. + +[pkg-mgr-example]: https://github.com/bazelbuild/examples/tree/main/bzlmod/05-integrate_third_party_package_manager + +### Detect toolchains on the host machine {:#detect-toolchain} + +When Bazel build rules need to detect what toolchains are available on your host +machine, they use repository rules to inspect the host machine and generate +toolchain info as external repositories. + +* **WORKSPACE** + + Given the following repository rule to detect a shell toolchain. + + ```python + ## local_config_sh.bzl + def _sh_config_rule_impl(repository_ctx): + sh_path = get_sh_path_from_env("SH_BIN_PATH") + + if not sh_path: + sh_path = detect_sh_from_path() + + if not sh_path: + sh_path = "/shell/binary/not/found" + + repository_ctx.file("BUILD", """ + load("@bazel_tools//tools/sh:sh_toolchain.bzl", "sh_toolchain") + sh_toolchain( + name = "local_sh", + path = "{sh_path}", + visibility = ["//visibility:public"], + ) + toolchain( + name = "local_sh_toolchain", + toolchain = ":local_sh", + toolchain_type = "@bazel_tools//tools/sh:toolchain_type", + ) + """.format(sh_path = sh_path)) + + sh_config_rule = repository_rule( + environ = ["SH_BIN_PATH"], + local = True, + implementation = _sh_config_rule_impl, + ) + ``` + + You can load the repository rule in WORKSPACE. + + ```python + ## WORKSPACE + load("//:local_config_sh.bzl", "sh_config_rule") + sh_config_rule(name = "local_config_sh") + ``` + +* **Bzlmod** + + With Bzlmod, you can introduce the same repository using a module extension, + which is similar to introducing the `@data_file` repository in the last + section. + + ``` + ## local_config_sh_extension.bzl + load("//:local_config_sh.bzl", "sh_config_rule") + + sh_config_extension = module_extension( + implementation = lambda ctx: sh_config_rule(name = "local_config_sh"), + ) + ``` + + Then use the extension in the MODULE.bazel file. + + ```python + ## MODULE.bazel + sh_config_ext = use_extension("//:local_config_sh_extension.bzl", "sh_config_extension") + use_repo(sh_config_ext, "local_config_sh") + ``` + +### Register toolchains & execution platforms {:#register-toolchains} + +Following the last section, after introducing a repository hosting toolchain +information (e.g. `local_config_sh`), you probably want to register the +toolchain. + +* **WORKSPACE** + + With WORKSPACE, you can register the toolchain in the following ways. + + 1. You can register the toolchain the `.bzl` file and load the macro in the + WORKSPACE file. + + ```python + ## local_config_sh.bzl + def sh_configure(): + sh_config_rule(name = "local_config_sh") + native.register_toolchains("@local_config_sh//:local_sh_toolchain") + ``` + + ```python + ## WORKSPACE + load("//:local_config_sh.bzl", "sh_configure") + sh_configure() + ``` + + 2. Or register the toolchain in the WORKSPACE file directly. + + ```python + ## WORKSPACE + load("//:local_config_sh.bzl", "sh_config_rule") + sh_config_rule(name = "local_config_sh") + register_toolchains("@local_config_sh//:local_sh_toolchain") + ``` + +* **Bzlmod** + + With Bzlmod, the + [`register_toolchains`](/rules/lib/globals/module#register_toolchains) and + [`register_execution_platforms`][register_execution_platforms] + APIs are only available in the MODULE.bazel file. You cannot call + `native.register_toolchains` in a module extension. + + ```python + ## MODULE.bazel + sh_config_ext = use_extension("//:local_config_sh_extension.bzl", "sh_config_extension") + use_repo(sh_config_ext, "local_config_sh") + register_toolchains("@local_config_sh//:local_sh_toolchain") + ``` + +[register_execution_platforms]: /rules/lib/globals/module#register_execution_platforms + +### Introduce local repositories {:#introduce-local-deps} + +You may need to introduce a dependency as a local repository when you need a +local version of the dependency for debugging or you want to incorporate a +directory in your workspace as external repository. + +* **WORKSPACE** + + With WORKSPACE, this is achieved by two native repository rules, + [`local_repository`](/reference/be/workspace#local_repository) and + [`new_local_repository`](/reference/be/workspace#new_local_repository). + + ```python + ## WORKSPACE + local_repository( + name = "rules_java", + path = "/Users/bazel_user/workspace/rules_java", + ) + ``` + +* **Bzlmod** + + With Bzlmod, you can use + [`local_path_override`](/rules/lib/globals/module#local_path_override) to + override a module with a local path. + + ```python + ## MODULE.bazel + bazel_dep(name = "rules_java") + local_path_override( + module_name = "rules_java", + path = "/Users/bazel_user/workspace/rules_java", + ) + ``` + + Note: With `local_path_override`, you can only introduce a local directory + as a Bazel module, which means it should have a MODULE.bazel file and its + transitive dependencies are taken into consideration during dependency + resolution. In addition, all module override directives can only be used by + the root module. + + It is also possible to introduce a local repository with module extension. + However, you cannot call `native.local_repository` in module extension, + there is ongoing effort on starlarkifying all native repository rules (check + [#18285](https://github.com/bazelbuild/bazel/issues/18285) for progress). + Then you can call the corresponding starlark `local_repository` in a module + extension. It's also trivial to implement a custom version of + `local_repository` repository rule if this is a blocking issue for you. + +### Bind targets {:#bind-targets} + +The [`bind`](/reference/be/workspace#bind) rule in WORKSPACE is deprecated and +not supported in Bzlmod. It was introduced to give a target an alias in the +special `//external` package. All users depending on this should migrate away. + +For example, if you have + +```python +## WORKSPACE +bind( + name = "openssl", + actual = "@my-ssl//src:openssl-lib", +) +``` + +This allows other targets to depend on `//external:openssl`. You can migrate +away from this by: + +* Replace all usages of `//external:openssl` with + `@my-ssl//src:openssl-lib`. + +* Or use the [`alias`](/reference/be/general#alias) build rule + * Define the following target in a package (e.g. `//third_party`) + + ```python + ## third_party/BUILD + alias( + name = "openssl, + actual = "@my-ssl//src:openssl-lib", + ) + ``` + + * Replace all usages of `//external:openssl` with + `//third_party:openssl-lib`. + +## Migration {:#migration} + +This section provides useful information and guidance for your Bzlmod migration +process. + +### Know your dependencies in WORKSPACE {:#know-deps-in-workspace} + +The first step of migration is to understand what dependencies you have. It +could be hard to figure out what exact dependencies are introduced in the +WORKSPACE file because transitive dependencies are often loaded with `*_deps` +macros. + +#### Inspect external dependency with workspace resolved file + +Fortunately, the flag +[`--experimental_repository_resolved_file`][resolved_file_flag] +can help. This flag essentially generates a "lock file" of all fetched external +dependencies in your last Bazel command. You can find more details in this [blog +post](https://blog.bazel.build/2018/07/09/bazel-sync-and-resolved-file.html). + +[resolved_file_flag]: /reference/command-line-reference#flag--experimental_repository_resolved_file + +It can be used in two ways: + +1. To fetch info of external dependencies needed for building certain targets. + + ```shell + bazel clean --expunge + bazel build --nobuild --experimental_repository_resolved_file=resolved.bzl //foo:bar + ``` + +2. To fetch info of all external dependencies defined in the WORKSPACE file. + + ```shell + bazel clean --expunge + bazel sync --experimental_repository_resolved_file=resolved.bzl + ``` + + With the `bazel sync` command, you can fetch all dependencies defined in the + WORKSPACE file, which include: + + * `bind` usages + * `register_toolchains` & `register_execution_platforms` usages + + However, if your project is cross platforms, bazel sync may break on certain + platforms because some repository rules may only run correctly on supported + platforms. + +After running the command, you should have information of your external +dependencies in the `resolved.bzl` file. + +#### Inspect external dependency with `bazel query` + +You may also know `bazel query` can be used for inspecting repository rules with + +```shell +bazel query --output=build //external: +``` + +While it is more convenient and much faster, [bazel query can lie about +external dependency version](https://github.com/bazelbuild/bazel/issues/12947), +so be careful using it! Querying and inspecting external +dependencies with Bzlmod is going to achieved by a [new +subcommand](https://github.com/bazelbuild/bazel/issues/15365). + +#### Built-in default dependencies {:#builtin-default-deps} + +If you check the file generated by `--experimental_repository_resolved_file`, +you are going to find many dependencies that are not defined in your WORKSPACE. +This is because Bazel in fact adds prefixes and suffixes to the user's WORKSPACE +file content to inject some default dependencies, which are usually required by +native rules (e.g. `@bazel_tools`, `@platforms` and `@remote_java_tools`). With +Bzlmod, those dependencies are introduced with a built-in module +[`bazel_tools`][bazel_tools] , which is a default dependency for every other +Bazel module. + +[bazel_tools]: https://github.com/bazelbuild/bazel/blob/master/src/MODULE.tools + +### Hybrid mode for gradual migration {:#hybrid-mode} + +Bzlmod and WORKSPACE can work side by side, which allows migrating dependencies +from the WORKSPACE file to Bzlmod to be a gradual process. + +#### WORKSPACE.bzlmod {:#workspace.bzlmod} + +During the migration, Bazel users may need to switch between builds with and +without Bzlmod enabled. WORKSPACE.bzlmod support is implemented to make the +process smoother. + +WORKSPACE.bzlmod has the exact same syntax as WORKSPACE. When Bzlmod is enabled, +if a WORKSPACE.bzlmod file also exists at the workspace root: + +* `WORKSPACE.bzlmod` takes effect and the content of `WORKSPACE` is ignored. +* No [prefixes or suffixes](/external/migration#builtin-default-deps) are + added to the WORKSPACE.bzlmod file. + +Using the WORKSPACE.bzlmod file can make the migration easier because: + +* When Bzlmod is disabled, you fall back to fetching dependencies from the + original WORKSPACE file. +* When Bzlmod is enabled, you can better track what dependencies are left to + migrate with WORKSPACE.bzlmod. + +Note: WORKSPACE.bzlmod does NOT replace the functionality of WORKSPACE for +identifying the workspace root, therefore you still need a WORKSPACE file at +your workspace root. + +#### Repository visibility {:#repository-visibility} + +Bzlmod is able to control which other repositories are visible from a given +repository, check [repository names and strict +deps](/external/module#repository_names_and_strict_deps) for more details. + +Here is a summary of repository visibilities from different types of +repositories when also taking WORKSPACE into consideration. + +| | From the main repo | From Bazel module repos | From module extension repos | From WORKSPACE repos | +|----------------|--------------------|-------------------------|---------------------------------------------------------------------------------------------------------------------|----------------------| +| The main repo | Visible | If the root module is a direct dependency | If the root module is a direct dependency of the module hosting the module extension | Visible | +| Bazel module repos | Direct deps | Direct deps | Direct deps of the module hosting the module extension | Direct deps of the root module | +| Module extension repos | Direct deps | Direct deps | Direct deps of the module hosting the module extension + all repos generated by the same module extension | Direct deps of the root module | +| WORKSPACE Repos | All visible | Not visible | Not visible | All visible | + +Note: For the root module, if a repository `@foo` is defined in WORKSPACE and +`@foo` is also used as an [apparent repository +name](/external/overview#apparent-repo-name) in MODULE.bazel, then `@foo` +refers to the one introduced in MODULE.bazel. + +Note: For a module extension generated repository `@bar`, if `@foo` is used as +an [apparent repository name](/external/overview#apparent-repo-name) of +another repository generated by the same module extension and direct +dependencies of the module hosting the module extension, then for repository +`@bar`, `@foo` refers to the latter. + +### Migration process {:#migration-process} + +A typical Bzlmod migration process can look like this: + +1. Understand what dependencies you have in WORKSPACE. +1. Add an empty MODULE.bazel file at your project root. +1. Add an empty WORKSPACE.bzlmod file to override the WORKSPACE file content. +1. Build your targets with Bzlmod enabled and check which repository is + missing. +1. Check the definition of the missing repository in the resolved dependency + file. +1. Introduce the missing dependency as a Bazel module, through a module + extension, or leave it in the WORKSPACE.bzlmod for later migration. +1. Go back to 4 and repeat until all dependencies are available. + +#### Migration tool {:#migration-tool} + +There is an interactive Bzlmod migration [helper script][migration_script] that +can get you started. + +[migration_script]: https://github.com/bazelbuild/bazel-central-registry/blob/main/tools/migrate_to_bzlmod.py + +The script does the following things: + +* Generate and parse the WORKSPACE resolved file. +* Print repository info from the resolved file in a human readable way. +* Run bazel build command, detect recognized error messages, and recommend a + way to migrate. +* Check if a dependency is already available in the BCR. +* Add a dependency to MODULE.bazel file. +* Add a dependency through a module extension. +* Add a dependency to WORKSPACE.bzlmod file. + +To use it, make sure you have the latest Bazel release installed, and run the +following command: + +```shell +git clone https://github.com/bazelbuild/bazel-central-registry.git +cd +/tools/migrate_to_bzlmod.py -t +``` + +Note: The migration script is not perfect and may not be up-to-date since Bzlmod +is evolving, always double check if the recommended solution is correct. + +## Publish Bazel modules {:#publish-modules} + +If your Bazel project is a dependency for other projects, you can publish your +project in the [Bazel Central Registry](https://registry.bazel.build/). + +To be able to check in your project in the BCR, you need a source archive URL of +the project. Take note of a few things when creating the source archive: + +* **Make sure the archive is pointing to a specific version.** + + The BCR can only accept versioned source archives because Bzlmod needs to + conduct version comparison during dependency resolution. + +* **Make sure the archive URL is stable.** + + Bazel verifies the content of the archive by a hash value, so you should + make sure the checksum of the downloaded file never changes. If the URL is + from GitHub, please create and upload a release archive in the release page. + GitHub isn't going to guarantee the checksum of source archives generated on + demand. In short, URLs in the form of + `https://github.com///releases/download/...` is considered stable + while `https://github.com///archive/...` is not. Check [GitHub + Archive Checksum + Outage](https://blog.bazel.build/2023/02/15/github-archive-checksum.html) + for more context. + +* **Make sure the source tree follows the layout of the original repository.** + + In case your repository is very large and you want to create a distribution + archive with reduced size by stripping out unnecessary sources, please make + sure the stripped source tree is a subset of the original source tree. This + makes it easier for end users to override the module to a non-release + version by [`archive_override`](/rules/lib/globals/module#archive_override) + and [`git_override`](/rules/lib/globals/module#git_override). + +* **Include a test module in a subdirectory that tests your most common + APIs.** + + A test module is a Bazel project with its own WORKSPACE and MODULE.bazel + file located in a subdirectory of the source archive which depends on the + actual module to be published. It should contain examples or some + integration tests that cover your most common APIs. Check + [test module][test_module] to learn how to set it up. + +[test_module]: https://github.com/bazelbuild/bazel-central-registry/tree/main/docs#test-module + +When you have your source archive URL ready, follow the [BCR contribution +guidelines][bcr_contrib_guide] to submit your module to the BCR with a GitHub +Pull Request. + +[bcr_contrib_guide]: https://github.com/bazelbuild/bazel-central-registry/tree/main/docs#contribute-a-bazel-module + +It is **highly recommended** to set up the [Publish to +BCR](https://github.com/bazel-contrib/publish-to-bcr) GitHub App for your +repository to automate the process of submitting your module to the BCR. + +## Best practices {:#best-practices} + +This section documents a few best practices you should follow for better +managing your external dependencies. + +#### Split targets into different packages to avoid fetching unnecessary dependencies. + +Check [#12835](https://github.com/bazelbuild/bazel/issues/12835), where dev +dependencies for tests are forced to be fetched unnecessarily for building +targets that don't need them. This is not actually not Bzlmod specific, but +following this practices makes it easier to specify dev dependencies correctly. + +#### Specify dev dependencies + +You can set the `dev_dependency` attribute to true for +[`bazel_dep`](/rules/lib/globals/module#bazel_dep) and +[`use_extension`](/rules/lib/globals/module#use_extension) directives so that +they don't propagate to dependent projects. As the root module, you can use the +[`--ignore_dev_dependency`][ignore_dev_dep_flag] flag to verify if your targets +still build without dev dependencies. + +[ignore_dev_dep_flag]: /reference/command-line-reference#flag--ignore_dev_dependency + +{# More best practices here !!! #} + +## Community migration progress {:#migration-progress} + +You can check the [Bazel Central Registry](https://registry.bazel.build) to find +out if your dependencies are already available. Otherwise feel free to join this +[GitHub discussion](https://github.com/bazelbuild/bazel/discussions/18329) to +upvote or post the dependencies that are blocking your migration. + +## Report issues {:#reporting-issues} + +Please check the [Bazel GitHub issue list][bzlmod_github_issue] for known Bzlmod +issues. Feel free to file new issues or feature requests that can help unblock +your migration! + +[bzlmod_github_issue]: https://github.com/bazelbuild/bazel/issues?q=is%3Aopen+is%3Aissue+label%3Aarea-Bzlmod diff --git a/site/en/external/module.md b/site/en/external/module.md index c07a299aa041d5..eda5fc0177da39 100644 --- a/site/en/external/module.md +++ b/site/en/external/module.md @@ -175,16 +175,16 @@ Bazel supports the following non-registry overrides: ## Repository names and strict deps -The [canonical name](/external/overview#canonical_repository_name) of a repo -backing a module is `{{ "" }}module_name{{ "" }}~{{ "" -}}version{{ "" }}` (for example, `bazel_skylib~1.0.3`). For modules with a +The [canonical name](/external/overview#canonical-repo-name) of a repo backing a +module is `{{ "" }}module_name{{ "" }}~{{ "" }}version{{ +"" }}` (for example, `bazel_skylib~1.0.3`). For modules with a non-registry override, replace the `{{ "" }}version{{ "" }}` part with the string `override`. Note that the canonical name format is not an API you should depend on and is subject to change at any time. -The [apparent name](/external/overview#apparent_repository_name) of a repo -backing a module to its direct dependents defaults to its module name, unless -the `repo_name` attribute of the [`bazel_dep`](/rules/lib/globals/module#bazel_dep) +The [apparent name](/external/overview#apparent-repo-name) of a repo backing a +module to its direct dependents defaults to its module name, unless the +`repo_name` attribute of the [`bazel_dep`](/rules/lib/globals/module#bazel_dep) directive says otherwise. Note that this means a module can only find its direct dependencies. This helps prevent accidental breakages due to changes in transitive dependencies. diff --git a/site/en/external/overview.md b/site/en/external/overview.md index c2511ecfc76bc9..97714878a0f46e 100644 --- a/site/en/external/overview.md +++ b/site/en/external/overview.md @@ -15,23 +15,24 @@ the traditional, repository-focused [`WORKSPACE` system](#workspace-system), and the newer module-focused [`MODULE.bazel` system](#bzlmod) (codenamed *Bzlmod*, and enabled with the flag `--enable_bzlmod`). The two systems can be used together, but Bzlmod is replacing the `WORKSPACE` system in future Bazel -releases. +releases, check the [Bzlmod migration guide](/external/migration) on how to +migrate. -This article explains the concepts surrounding external dependency management in -Bazel, before going into a bit more detail about the two systems in order. +This document explains the concepts surrounding external dependency management +in Bazel, before going into a bit more detail about the two systems in order. -## Concepts +## Concepts {:#concepts} -### Repository +### Repository {:#repository} A directory with a `WORKSPACE` or `WORKSPACE.bazel` file, containing source files to be used in a Bazel build. Often shortened to just **repo**. -### Main repository +### Main repository {:#main-repository} The repository in which the current Bazel command is being run. -### Workspace +### Workspace {:#workspace} The environment shared by all Bazel commands run in the same main repository. @@ -39,7 +40,7 @@ Note that historically the concepts of "repository" and "workspace" have been conflated; the term "workspace" has often been used to refer to the main repository, and sometimes even used as a synonym of "repository". -### Canonical repository name +### Canonical repository name {:#canonical-repo-name} The canonical name a repository is addressable by. Within the context of a workspace, each repository has a single canonical name. A target inside a repo @@ -48,7 +49,7 @@ whose canonical name is `canonical_name` can be addressed by the label The main repository always has the empty string as the canonical name. -### Apparent repository name +### Apparent repository name {:#apparent-repo-name} The name a repository is addressable by in the context of a certain other repo. This can be thought of as a repo's "nickname": The repo with the canonical name @@ -60,7 +61,7 @@ This can be thought of as a repo's "nickname": The repo with the canonical name Conversely, this can be understood as a **repository mapping**: each repo maintains a mapping from "apparent repo name" to a "canonical repo name". -### Repository rule +### Repository rule {:#repo-rule} A schema for repository definitions that tells Bazel how to materialize a repository. For example, it could be "download a zip archive from a certain URL @@ -68,7 +69,7 @@ and extract it", or "fetch a certain Maven artifact and make it available as a `java_import` target", or simply "symlink a local directory". Every repo is **defined** by calling a repo rule with an appropriate number of arguments. -See [Repository rules](/extending/repo) for more information on how to write +See [Repository rules](/extending/repo) for more information about how to write your own repository rules. The most common repo rules by far are @@ -77,17 +78,17 @@ from a URL and extracts it, and [`local_repository`](/reference/be/workspace#local_repository), which symlinks a local directory that is already a Bazel repository. -### Fetching a repository +### Fetch a repository {:#fetch-repository} The action of making a repo available on local disk by running its associated repo rule. The repos defined in a workspace are not available on local disk before they are fetched. -Normally, Bazel will only fetch a repo when it needs something from the repo, +Normally, Bazel only fetches a repo when it needs something from the repo, and the repo hasn't already been fetched. If the repo has already been fetched -before, Bazel will only re-fetch it if its definition has changed. +before, Bazel only re-fetches it if its definition has changed. -### Directory layout +### Directory layout {:#directory-layout} After being fetched, the repo can be found in the subdirectory `external` in the [output base](/remote/output-directories), under its canonical name. @@ -99,7 +100,7 @@ canonical name `canonical_name`: ls $(bazel info output_base)/external/{{ '' }} canonical_name {{ '' }} ``` -## Managing external dependencies with Bzlmod {:#bzlmod} +## Manage external dependencies with Bzlmod {:#bzlmod} Bzlmod, the new external dependency subsystem, does not directly work with repo definitions. Instead, it builds a dependency graph from _modules_, runs @@ -137,19 +138,18 @@ requests. Among other things, they allow Bazel to interact with other package management systems while also respecting the dependency graph built out of Bazel modules. -### External links on Bzlmod +### External links on Bzlmod {:#external-links} -* [Bzlmod Migration Guide](https://docs.google.com/document/d/1JtXIVnXyFZ4bmbiBCr5gsTH4-opZAFf5DMMb-54kES0/edit?usp=sharing){:.external} * [Bzlmod usage examples in bazelbuild/examples](https://github.com/bazelbuild/examples/tree/main/bzlmod){:.external} * [Bazel External Dependencies Overhaul](https://docs.google.com/document/d/1moQfNcEIttsk6vYanNKIy3ZuK53hQUFq1b1r0rmsYVg/edit){: .external} (original Bzlmod design doc) * [BazelCon 2021 talk on Bzlmod](https://www.youtube.com/watch?v=TxOCKtU39Fs){: .external} * [Bazel Community Day talk on Bzlmod](https://www.youtube.com/watch?v=MB6xxis9gWI){: .external} -## Defining repos with `WORKSPACE` {:#workspace-system} +## Define repos with `WORKSPACE` {:#workspace-system} -Historically, you can manage external dependencies by defining repos in -the `WORKSPACE` (or `WORKSPACE.bazel`) file. This file has a similar syntax to +Historically, you can manage external dependencies by defining repos in the +`WORKSPACE` (or `WORKSPACE.bazel`) file. This file has a similar syntax to `BUILD` files, employing repo rules instead of build rules. The following snippet is an example to use the `http_archive` repo rule in the @@ -168,9 +168,10 @@ The snippet defines a repo whose canonical name is `foo`. In the `WORKSPACE` system, by default, the canonical name of a repo is also its apparent name to all other repos. -See the [full list](/rules/lib/globals/workspace) of functions available in `WORKSPACE` files. +See the [full list](/rules/lib/globals/workspace) of functions available in +`WORKSPACE` files. -### Shortcomings of the `WORKSPACE` system +### Shortcomings of the `WORKSPACE` system {:#workspace-shortcomings} In the years since the `WORKSPACE` system was introduced, users have reported many pain points, including: @@ -189,4 +190,8 @@ many pain points, including: dependencies are specified using `http_archive` with URLs, without any version information. This means that there is no reliable way to perform version resolution in the case of diamond dependencies (`A` depends on - `B` and `C`; `B` and `C` both depend on different versions of `D`). \ No newline at end of file + `B` and `C`; `B` and `C` both depend on different versions of `D`). + +Due to the shortcomings of WORKSPACE, Bzlmod is going to replace the legacy +WORKSPACE system in future Bazel releases. Please read the [Bzlmod migration +guide](/external/migration) on how to migrate to Bzlmod. \ No newline at end of file diff --git a/site/en/help.md b/site/en/help.md index 3b8671f33cd72f..f9bf54b4c5ac6f 100644 --- a/site/en/help.md +++ b/site/en/help.md @@ -44,6 +44,11 @@ If there are no existing answers, you can ask the community by: * Chatting with other Bazel contributors on [Slack](https://slack.bazel.build/) * Consulting a [Bazel community expert](/community/experts) +## Understand Bazel's support level {:#support-level} + +Please read the [release page](/release) to understand Bazel's release model and +what level of support Bazel provides. + ## File a bug {:#file-bug} If you encounter a bug or want to request a feature, file a [GitHub diff --git a/site/en/remote/multiplex.md b/site/en/remote/multiplex.md index 9491289a6b2e6c..f7c0bee538b2c3 100644 --- a/site/en/remote/multiplex.md +++ b/site/en/remote/multiplex.md @@ -73,7 +73,7 @@ strategy needs to be specified, either at the ruleset level (for example, example, `--dynamic_local_strategy=worker,standalone`.) No additional flags are necessary, and `supports-multiplex-workers` takes precedence over `supports-workers`, if both are set. You can turn off multiplex workers -globally by passing `--noexperimental_worker_multiplex`. +globally by passing `--noworker_multiplex`. A ruleset is encouraged to use multiplex workers if possible, to reduce memory pressure and improve performance. However, multiplex workers are not currently diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipTester.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipTester.java index f2e8b3a8445dd6..ac18f593e4dd4d 100644 --- a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipTester.java +++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipTester.java @@ -228,7 +228,7 @@ private void readEntry() throws IOException { String name = new String(filename, "UTF-8"); for (int i = 0; i < filename.length; i++) { - if ((filename[i] < ' ') || (filename[i] > 127)) { + if ((filename[i] < ' ')) { warn(entryDesc + ": file name has unexpected non-ascii characters"); } } diff --git a/src/jdeps_modules.golden b/src/jdeps_modules.golden index 4584f9d392fdfc..5aa0d6085a8ad3 100644 --- a/src/jdeps_modules.golden +++ b/src/jdeps_modules.golden @@ -7,5 +7,4 @@ java.naming java.sql java.xml jdk.management -jdk.sctp jdk.unsupported diff --git a/src/main/java/com/google/devtools/build/docgen/templates/be/make-variables.vm b/src/main/java/com/google/devtools/build/docgen/templates/be/make-variables.vm index 26f536461b47c6..76911b1efadf92 100644 --- a/src/main/java/com/google/devtools/build/docgen/templates/be/make-variables.vm +++ b/src/main/java/com/google/devtools/build/docgen/templates/be/make-variables.vm @@ -412,7 +412,7 @@

These variables are a fallback mechanism to be used by language experts in rare cases. If you are tempted to use them, please contact the Bazel devs first. + href="https://bazel.build/help">contact the Bazel devs first.

    @@ -463,7 +463,7 @@

    These variables are a fallback mechanism to be used by language experts in rare cases. If you are tempted to use them, please contact the Bazel devs first. + href="https://bazel.build/help">contact the Bazel devs first.

      diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionCacheUtils.java b/src/main/java/com/google/devtools/build/lib/actions/ActionCacheUtils.java index 49065220a98869..a7e6b7bd93fe4a 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/ActionCacheUtils.java +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionCacheUtils.java @@ -14,12 +14,26 @@ package com.google.devtools.build.lib.actions; import com.google.devtools.build.lib.actions.cache.ActionCache; +import java.util.AbstractMap.SimpleEntry; +import java.util.Map.Entry; import javax.annotation.Nullable; /** Utility functions for {@link ActionCache}. */ public class ActionCacheUtils { private ActionCacheUtils() {} + @Nullable + public static Entry getCacheEntryWithKey( + ActionCache actionCache, Action action) { + for (Artifact output : action.getOutputs()) { + ActionCache.Entry entry = actionCache.get(output.getExecPathString()); + if (entry != null) { + return new SimpleEntry<>(output.getExecPathString(), entry); + } + } + return null; + } + /** Checks whether one of existing output paths is already used as a key. */ @Nullable public static ActionCache.Entry getCacheEntry(ActionCache actionCache, Action action) { diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionLookupKeyOrProxy.java b/src/main/java/com/google/devtools/build/lib/actions/ActionLookupKeyOrProxy.java index 48f76839214673..92b189cf5e8c3b 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/ActionLookupKeyOrProxy.java +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionLookupKeyOrProxy.java @@ -29,7 +29,6 @@ * are subclasses of {@link ActionLookupKeyOrProxy}. This allows callers to easily find the value * key, while remaining agnostic to what action lookup values actually exist. */ -// TODO(b/261521010): this layer of indirection is no longer needed and may be cleaned up. public interface ActionLookupKeyOrProxy extends ArtifactOwner { /** * Returns the {@link BuildConfigurationKey} for the configuration associated with this key, or diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionOwner.java b/src/main/java/com/google/devtools/build/lib/actions/ActionOwner.java index 643ee3591b9b5b..65f487cbd739e8 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/ActionOwner.java +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionOwner.java @@ -42,7 +42,7 @@ public abstract class ActionOwner { /* label= */ null, Location.BUILTIN, /* targetKind= */ "empty target kind", - /* mnemonic= */ "system", + /* buildConfigurationMnemonic= */ "system", /* configurationChecksum= */ "system", /* buildConfigurationEvent= */ null, /* isToolConfiguration= */ false, @@ -78,7 +78,7 @@ public static ActionOwner createDummy( @Nullable Label label, Location location, String targetKind, - String mnemonic, + String buildConfigurationMnemonic, String configurationChecksum, @Nullable BuildConfigurationEvent buildConfigurationEvent, boolean isToolConfiguration, @@ -90,7 +90,10 @@ public static ActionOwner createDummy( location, targetKind, BuildConfigurationInfo.AutoBuildConfigurationInfo.create( - mnemonic, configurationChecksum, buildConfigurationEvent, isToolConfiguration), + buildConfigurationMnemonic, + configurationChecksum, + buildConfigurationEvent, + isToolConfiguration), executionPlatform, aspectDescriptors, execProperties); @@ -112,7 +115,7 @@ public static ActionOwner createDummy( public abstract BuildConfigurationInfo getBuildConfigurationInfo(); /** Returns the mnemonic for the configuration for this {@link ActionOwner}. */ - public final String getMnemonic() { + public final String getBuildConfigurationMnemonic() { return getBuildConfigurationInfo().getMnemonic(); } diff --git a/src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java b/src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java index 39936bb605002d..44413c283563b6 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java +++ b/src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java @@ -627,6 +627,12 @@ public long getExpireAtEpochMilli() { return -1; } + /** + * Extends the expiration time for this metadata. If it was constructed without known expiration + * time (i.e. expireAtEpochMilli < 0), this extension does nothing. + */ + public void extendExpireAtEpochMilli(long expireAtEpochMilli) {} + public boolean isAlive(Instant now) { return true; } @@ -655,7 +661,7 @@ public final String toString() { /** A remote artifact that expires at a particular time. */ private static final class RemoteFileArtifactValueWithExpiration extends RemoteFileArtifactValue { - private final long expireAtEpochMilli; + private long expireAtEpochMilli; private RemoteFileArtifactValueWithExpiration( byte[] digest, @@ -672,6 +678,12 @@ public long getExpireAtEpochMilli() { return expireAtEpochMilli; } + @Override + public void extendExpireAtEpochMilli(long expireAtEpochMilli) { + Preconditions.checkState(expireAtEpochMilli > this.expireAtEpochMilli); + this.expireAtEpochMilli = expireAtEpochMilli; + } + @Override public boolean isAlive(Instant now) { return now.toEpochMilli() < expireAtEpochMilli; diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/PersistentStringIndexer.java b/src/main/java/com/google/devtools/build/lib/actions/cache/PersistentStringIndexer.java index 8cdc4f818a2c47..e12826c025d655 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/cache/PersistentStringIndexer.java +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/PersistentStringIndexer.java @@ -65,7 +65,7 @@ public PersistentIndexMap(Path mapFile, Path journalFile, Clock clock) throws IO @Override protected boolean updateJournal() { long time = clock.nanoTime(); - if (SAVE_INTERVAL_NS == 0 || time > nextUpdate) { + if (SAVE_INTERVAL_NS == 0L || time > nextUpdate) { nextUpdate = time + SAVE_INTERVAL_NS; return true; } diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AspectAwareAttributeMapper.java b/src/main/java/com/google/devtools/build/lib/analysis/AspectAwareAttributeMapper.java index 3ea773f1a6879e..d61e714daab92b 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/AspectAwareAttributeMapper.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/AspectAwareAttributeMapper.java @@ -20,6 +20,7 @@ import com.google.devtools.build.lib.packages.Attribute; import com.google.devtools.build.lib.packages.AttributeMap; import com.google.devtools.build.lib.packages.DependencyFilter; +import com.google.devtools.build.lib.packages.PackageArgs; import com.google.devtools.build.lib.packages.Type; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -137,23 +138,8 @@ public void visitLabels(DependencyFilter filter, BiConsumer co } @Override - public String getPackageDefaultHdrsCheck() { - return ruleAttributes.getPackageDefaultHdrsCheck(); - } - - @Override - public boolean isPackageDefaultHdrsCheckSet() { - return ruleAttributes.isPackageDefaultHdrsCheckSet(); - } - - @Override - public Boolean getPackageDefaultTestOnly() { - return ruleAttributes.getPackageDefaultTestOnly(); - } - - @Override - public String getPackageDefaultDeprecation() { - return ruleAttributes.getPackageDefaultDeprecation(); + public PackageArgs getPackageArgs() { + return ruleAttributes.getPackageArgs(); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BUILD b/src/main/java/com/google/devtools/build/lib/analysis/BUILD index 32d208771ecb0c..a7ff85cb5efa80 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/BUILD +++ b/src/main/java/com/google/devtools/build/lib/analysis/BUILD @@ -1048,6 +1048,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/actions:commandline_item", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/collect/nestedset", + "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/packages", "//src/main/java/com/google/devtools/build/lib/util", "//src/main/java/net/starlark/java/eval", @@ -1400,6 +1401,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/actions", "//src/main/java/com/google/devtools/build/lib/actions:artifacts", "//src/main/java/com/google/devtools/build/lib/collect/nestedset", + "//src/main/java/com/google/devtools/build/lib/events", "//third_party:guava", "//third_party:jsr305", ], diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BaseRuleClasses.java b/src/main/java/com/google/devtools/build/lib/analysis/BaseRuleClasses.java index 4045acc3cf1b7c..ac600e3f5d4f34 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/BaseRuleClasses.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/BaseRuleClasses.java @@ -68,7 +68,7 @@ private BaseRuleClasses() {} new Attribute.ComputedDefault() { @Override public Object getDefault(AttributeMap rule) { - return rule.getPackageDefaultTestOnly(); + return rule.getPackageArgs().defaultTestOnly(); } @Override @@ -82,7 +82,7 @@ public boolean resolvableWithRawAttributes() { new Attribute.ComputedDefault() { @Override public Object getDefault(AttributeMap rule) { - return rule.getPackageDefaultDeprecation(); + return rule.getPackageArgs().defaultDeprecation(); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/analysis/DependencyResolver.java b/src/main/java/com/google/devtools/build/lib/analysis/DependencyResolver.java index 8080517b9d725b..9178f98318f4e4 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/DependencyResolver.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/DependencyResolver.java @@ -590,14 +590,14 @@ private static void visitRule( outgoingLabels, rule, RuleClass.COMPATIBLE_ENVIRONMENT_ATTR, - rule.getPackage().getDefaultCompatibleWith()); + rule.getPackage().getPackageArgs().defaultCompatibleWith()); } if (!rule.isAttributeValueExplicitlySpecified(RuleClass.RESTRICTED_ENVIRONMENT_ATTR)) { addExplicitDeps( outgoingLabels, rule, RuleClass.RESTRICTED_ENVIRONMENT_ATTR, - rule.getPackage().getDefaultRestrictedTo()); + rule.getPackage().getPackageArgs().defaultRestrictedTo()); } addToolchainDeps(toolchainContexts, outgoingLabels); diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RepoMappingManifestAction.java b/src/main/java/com/google/devtools/build/lib/analysis/RepoMappingManifestAction.java index 347999e4e5c1f5..ea80019fbbe203 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/RepoMappingManifestAction.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/RepoMappingManifestAction.java @@ -15,6 +15,7 @@ import static com.google.common.collect.ImmutableSortedMap.toImmutableSortedMap; import static java.nio.charset.StandardCharsets.ISO_8859_1; +import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Comparator.comparing; import com.google.common.collect.ImmutableSet; @@ -26,7 +27,6 @@ import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander; import com.google.devtools.build.lib.actions.CommandLineExpansionException; import com.google.devtools.build.lib.actions.CommandLineItem.MapFn; -import com.google.devtools.build.lib.actions.ExecException; import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction; import com.google.devtools.build.lib.analysis.actions.DeterministicWriter; import com.google.devtools.build.lib.cmdline.Label; @@ -35,8 +35,11 @@ import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.packages.Package; import com.google.devtools.build.lib.util.Fingerprint; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.PrintWriter; import java.util.Map.Entry; import java.util.UUID; @@ -44,7 +47,8 @@ import net.starlark.java.eval.EvalException; /** Creates a manifest file describing the repos and mappings relevant for a runfile tree. */ -public final class RepoMappingManifestAction extends AbstractFileWriteAction { +public final class RepoMappingManifestAction extends AbstractFileWriteAction + implements AbstractFileWriteAction.FileContentsProvider { private static final UUID MY_UUID = UUID.fromString("458e351c-4d30-433d-b927-da6cddd4737f"); @@ -119,9 +123,24 @@ protected void computeKey( fp.addString(workspaceName); } + /** + * Get the contents of a file internally using an in memory output stream. + * + * @return returns the file contents as a string. + */ @Override - public DeterministicWriter newDeterministicWriter(ActionExecutionContext ctx) - throws InterruptedException, ExecException { + public String getFileContents(@Nullable EventHandler eventHandler) throws IOException { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + newDeterministicWriter().writeOutputFile(stream); + return stream.toString(UTF_8); + } + + @Override + public DeterministicWriter newDeterministicWriter(ActionExecutionContext ctx) { + return newDeterministicWriter(); + } + + public DeterministicWriter newDeterministicWriter() { return out -> { PrintWriter writer = new PrintWriter(out, /* autoFlush= */ false, ISO_8859_1); diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java b/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java index f897c66994ae9d..bb2999a1833afb 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java @@ -224,7 +224,7 @@ private RuleContext( } private FeatureSet computeFeatures() { - FeatureSet pkg = rule.getPackage().getFeatures(); + FeatureSet pkg = rule.getPackage().getPackageArgs().features(); FeatureSet rule = attributes().has("features", Type.STRING_LIST) ? FeatureSet.parse(attributes().get("features", Type.STRING_LIST)) diff --git a/src/main/java/com/google/devtools/build/lib/analysis/SourceManifestAction.java b/src/main/java/com/google/devtools/build/lib/analysis/SourceManifestAction.java index 89827c5ca432c1..e469c0bbefe5a8 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/SourceManifestAction.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/SourceManifestAction.java @@ -55,7 +55,8 @@ *

      This action carefully avoids building the manifest content in memory because it can be large. */ @Immutable // if all ManifestWriter implementations are immutable -public final class SourceManifestAction extends AbstractFileWriteAction { +public final class SourceManifestAction extends AbstractFileWriteAction + implements AbstractFileWriteAction.FileContentsProvider { private static final String GUID = "07459553-a3d0-4d37-9d78-18ed942470f4"; @@ -193,7 +194,8 @@ public void writeOutputFile(OutputStream out, @Nullable EventHandler eventHandle * * @return returns the file contents as a string. */ - public String getFileContentsAsString(@Nullable EventHandler eventHandler) throws IOException { + @Override + public String getFileContents(@Nullable EventHandler eventHandler) throws IOException { ByteArrayOutputStream stream = new ByteArrayOutputStream(); writeOutputFile(stream, eventHandler); return stream.toString(UTF_8); @@ -201,7 +203,7 @@ public String getFileContentsAsString(@Nullable EventHandler eventHandler) throw @Override public String getStarlarkContent() throws IOException { - return getFileContentsAsString(null); + return getFileContents(null); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TransitiveDependencyState.java b/src/main/java/com/google/devtools/build/lib/analysis/TransitiveDependencyState.java index 111b352f5ab504..ac9c872ea089cf 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/TransitiveDependencyState.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/TransitiveDependencyState.java @@ -62,7 +62,7 @@ public final class TransitiveDependencyState { * *

      More ideally, those properties would be conveyed via providers of those dependencies, but * doing so would adversely affect resting heap usage whereas {@link ConfiguredTargetAndData} is - * ephemeral. Distributed implementations will include these properties in an extra provider. It + * ephemeral. Distributed implementations will include these properties in an extra providers. It * won't affect memory because the underlying package won't exist on the node loading it remotely. * *

      It's valid to obtain {@link Package}s of dependencies from this map instead of creating an diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/AbstractFileWriteAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/AbstractFileWriteAction.java index 4d2c82baf8b7fc..a0da40e1ad3ba0 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/actions/AbstractFileWriteAction.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/AbstractFileWriteAction.java @@ -25,6 +25,9 @@ import com.google.devtools.build.lib.actions.ExecException; import com.google.devtools.build.lib.actions.SpawnResult; import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.events.EventHandler; +import java.io.IOException; +import javax.annotation.Nullable; /** * Abstract Action to write to a file. @@ -104,4 +107,11 @@ public boolean isRemotable() { return true; } + /** + * This interface is used to get the contents of the file to output to aquery when using + * --include_file_write_contents. + */ + public interface FileContentsProvider { + String getFileContents(@Nullable EventHandler eventHandler) throws IOException; + } } diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteAction.java index 76f367962af42a..556c12c582b38a 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteAction.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteAction.java @@ -25,6 +25,7 @@ import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; import com.google.devtools.build.lib.collect.nestedset.Order; import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.util.Fingerprint; import com.google.devtools.build.lib.util.OnDemandString; import java.io.ByteArrayInputStream; @@ -49,7 +50,8 @@ * BinaryFileWriteAction}. */ @Immutable // if fileContents is immutable -public final class FileWriteAction extends AbstractFileWriteAction { +public final class FileWriteAction extends AbstractFileWriteAction + implements AbstractFileWriteAction.FileContentsProvider { private static final String GUID = "332877c7-ca9f-4731-b387-54f620408522"; @@ -206,6 +208,14 @@ public String toString() { } } + /** + * @see #getFilecontents() + */ + @Override + public String getFileContents(@Nullable EventHandler eventHandler) { + return getFileContents(); + } + /** * Returns the string contents to be written. * diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java index 8a38f2b4c257c9..ae7e57f08ae564 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java @@ -600,7 +600,7 @@ public static class Builder { public Builder() {} /** Creates a builder that is a copy of another builder. */ - Builder(Builder other) { + public Builder(Builder other) { this.toolsBuilder.addTransitive(other.toolsBuilder.build()); this.inputsBuilder.addTransitive(other.inputsBuilder.build()); this.outputs.addAll(other.outputs); @@ -657,7 +657,7 @@ public SpawnAction build(ActionOwner owner, BuildConfigurationValue configuratio } @CheckReturnValue - SpawnAction buildForActionTemplate(ActionOwner owner) { + public SpawnAction buildForActionTemplate(ActionOwner owner) { CommandLines.Builder result = CommandLines.builder(); if (executableArg != null) { result.addSingleArgument(executableArg); @@ -683,12 +683,7 @@ private SpawnAction buildSpawnAction( ActionEnvironment env) { NestedSet tools = toolsBuilder.build(); - // Tools are by definition a subset of the inputs, so make sure they're present there, too. - NestedSet inputsAndTools = - NestedSetBuilder.stableOrder() - .addTransitive(inputsBuilder.build()) - .addTransitive(tools) - .build(); + NestedSet inputsAndTools = getInputsAndTools(); return createSpawnAction( owner, @@ -788,6 +783,17 @@ public Builder addInputs(Iterable artifacts) { return this; } + /** + * Returns the inputs that the spawn action will depend on. Tools are by definition a subset of + * the inputs, so they are also present. + */ + public NestedSet getInputsAndTools() { + return NestedSetBuilder.stableOrder() + .addTransitive(inputsBuilder.build()) + .addTransitive(toolsBuilder.build()) + .build(); + } + /** Adds transitive inputs to this action. */ @CanIgnoreReturnValue public Builder addTransitiveInputs(NestedSet artifacts) { diff --git a/src/main/java/com/google/devtools/build/lib/analysis/producers/AttributeConfiguration.java b/src/main/java/com/google/devtools/build/lib/analysis/producers/AttributeConfiguration.java index 106f2ef3836c48..c927e3c38467a0 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/producers/AttributeConfiguration.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/producers/AttributeConfiguration.java @@ -28,13 +28,6 @@ enum Kind { * target is known, it should be verified to be a {@link PackageGroup}. */ VISIBILITY, - /** - * The configuration is null. - * - *

      This is only applied when the dependency is in the same package as the parent and it is - * not configurable. - */ - NULL_CONFIGURATION, /** * There is a single configuration. * @@ -53,8 +46,6 @@ enum Kind { abstract void visibility(); - abstract void nullConfiguration(); - abstract BuildConfigurationKey unary(); abstract ImmutableMap split(); @@ -62,7 +53,6 @@ enum Kind { public int count() { switch (kind()) { case VISIBILITY: - case NULL_CONFIGURATION: case UNARY: return 1; case SPLIT: @@ -75,10 +65,6 @@ static AttributeConfiguration ofVisibility() { return AutoOneOf_AttributeConfiguration.visibility(); } - static AttributeConfiguration ofNullConfiguration() { - return AutoOneOf_AttributeConfiguration.nullConfiguration(); - } - static AttributeConfiguration ofUnary(BuildConfigurationKey key) { return AutoOneOf_AttributeConfiguration.unary(key); } diff --git a/src/main/java/com/google/devtools/build/lib/analysis/producers/ConfigConditionsProducer.java b/src/main/java/com/google/devtools/build/lib/analysis/producers/ConfigConditionsProducer.java index 7b71751f9bd179..c9d91a559ce06a 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/producers/ConfigConditionsProducer.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/producers/ConfigConditionsProducer.java @@ -14,7 +14,7 @@ package com.google.devtools.build.lib.analysis.producers; import com.google.common.collect.ImmutableMap; -import com.google.devtools.build.lib.analysis.InconsistentNullConfigException; +import com.google.devtools.build.lib.analysis.InvalidVisibilityDependencyException; import com.google.devtools.build.lib.analysis.TargetAndConfiguration; import com.google.devtools.build.lib.analysis.TransitiveDependencyState; import com.google.devtools.build.lib.analysis.config.ConfigConditions; @@ -130,12 +130,17 @@ public void acceptConfiguredTargetAndDataError(ConfiguredValueCreationException } @Override - public void acceptConfiguredTargetAndDataError(InconsistentNullConfigException error) { - // A config label was evaluated with a null configuration. This should never happen as - // ConfigConditions are only present if the parent is a Rule, then always evaluated with the - // parent configuration. + public void acceptConfiguredTargetAndDataError(InvalidVisibilityDependencyException error) { + // After removing the rule transition from dependency resolution, a ConfiguredTargetKey in + // Skyframe with a null BuildConfigurationKey will only be used to request visibility + // dependencies. This will never be the case for ConfigConditions, which are always requested + // with the parent configuration. At the moment, nothing throws + // InvalidVisibilityDependencyException. + // + // TODO(b/261521010): update this comment once rule transitions are removed from dependency + // resolution. throw new IllegalArgumentException( - "ConfigCondition dependency should never be evaluated with a null configuration.", error); + "ConfigCondition dependency should never be marked visibility.", error); } private StateMachine constructConfigConditions(Tasks tasks, ExtendedEventHandler listener) { diff --git a/src/main/java/com/google/devtools/build/lib/analysis/producers/ConfiguredTargetAndDataProducer.java b/src/main/java/com/google/devtools/build/lib/analysis/producers/ConfiguredTargetAndDataProducer.java index 50c475a1824509..2ca4a755ff50b2 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/producers/ConfiguredTargetAndDataProducer.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/producers/ConfiguredTargetAndDataProducer.java @@ -16,7 +16,7 @@ import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.analysis.ConfiguredTarget; import com.google.devtools.build.lib.analysis.ConfiguredTargetValue; -import com.google.devtools.build.lib.analysis.InconsistentNullConfigException; +import com.google.devtools.build.lib.analysis.InvalidVisibilityDependencyException; import com.google.devtools.build.lib.analysis.TransitiveDependencyState; import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; import com.google.devtools.build.lib.events.ExtendedEventHandler; @@ -42,14 +42,14 @@ public final class ConfiguredTargetAndDataProducer implements StateMachine, Consumer, StateMachine.ValueOrException2Sink< - ConfiguredValueCreationException, InconsistentNullConfigException> { + ConfiguredValueCreationException, InvalidVisibilityDependencyException> { /** Interface for accepting values produced by this class. */ public interface ResultSink { void acceptConfiguredTargetAndData(ConfiguredTargetAndData value, int index); void acceptConfiguredTargetAndDataError(ConfiguredValueCreationException error); - void acceptConfiguredTargetAndDataError(InconsistentNullConfigException error); + void acceptConfiguredTargetAndDataError(InvalidVisibilityDependencyException error); } // -------------------- Input -------------------- @@ -86,8 +86,9 @@ public StateMachine step(Tasks tasks, ExtendedEventHandler listener) { tasks.lookUp( key.toKey(), ConfiguredValueCreationException.class, - InconsistentNullConfigException.class, - (ValueOrException2Sink) + InvalidVisibilityDependencyException.class, + (ValueOrException2Sink< + ConfiguredValueCreationException, InvalidVisibilityDependencyException>) this); return this::fetchConfigurationAndPackage; } @@ -96,7 +97,7 @@ public StateMachine step(Tasks tasks, ExtendedEventHandler listener) { public void acceptValueOrException2( @Nullable SkyValue value, @Nullable ConfiguredValueCreationException error, - @Nullable InconsistentNullConfigException visibilityError) { + @Nullable InvalidVisibilityDependencyException visibilityError) { if (value != null) { var configuredTargetValue = (ConfiguredTargetValue) value; this.configuredTarget = configuredTargetValue.getConfiguredTarget(); diff --git a/src/main/java/com/google/devtools/build/lib/analysis/producers/DependencyProducer.java b/src/main/java/com/google/devtools/build/lib/analysis/producers/DependencyProducer.java index e6bddce74310ef..2940409889ba9d 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/producers/DependencyProducer.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/producers/DependencyProducer.java @@ -30,16 +30,11 @@ import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; import com.google.devtools.build.lib.analysis.config.DependencyEvaluationException; import com.google.devtools.build.lib.analysis.starlark.StarlarkTransition.TransitionException; -import com.google.devtools.build.lib.causes.LoadingFailedCause; import com.google.devtools.build.lib.cmdline.Label; -import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.devtools.build.lib.packages.Aspect; import com.google.devtools.build.lib.packages.Attribute; import com.google.devtools.build.lib.packages.AttributeTransitionData; -import com.google.devtools.build.lib.packages.NoSuchTargetException; -import com.google.devtools.build.lib.packages.Target; -import com.google.devtools.build.lib.packages.TargetUtils; import com.google.devtools.build.lib.skyframe.BuildConfigurationKey; import com.google.devtools.build.lib.skyframe.ConfiguredTargetAndData; import com.google.devtools.build.lib.skyframe.ConfiguredValueCreationException; @@ -117,25 +112,6 @@ public StateMachine step(Tasks tasks, ExtendedEventHandler listener) { AttributeConfiguration.ofVisibility(), /* executionPlatformLabel= */ null); } - Target parentTarget = parameters.target(); - if (parentTarget.getLabel().getPackageIdentifier().equals(toLabel.getPackageIdentifier())) { - try { - Target toTarget = parentTarget.getPackage().getTarget(toLabel.getName()); - if (!toTarget.isConfigurable()) { - return computePrerequisites( - AttributeConfiguration.ofNullConfiguration(), /* executionPlatformLabel= */ null); - } - } catch (NoSuchTargetException e) { - parameters - .transitiveState() - .addTransitiveCause(new LoadingFailedCause(toLabel, e.getDetailedExitCode())); - listener.handle( - Event.error( - TargetUtils.getLocationMaybe(parentTarget), - TargetUtils.formatMissingEdge(parentTarget, toLabel, e, kind.getAttribute()))); - } - } - // The logic of `DependencyResolver.computeDependencyLabels` implies that // `parameters.configurationKey()` is non-null for everything that follows. BuildConfigurationKey configurationKey = checkNotNull(parameters.configurationKey()); diff --git a/src/main/java/com/google/devtools/build/lib/analysis/producers/PrerequisiteParameters.java b/src/main/java/com/google/devtools/build/lib/analysis/producers/PrerequisiteParameters.java index 7ab5fd1300888c..4cdd28db3de305 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/producers/PrerequisiteParameters.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/producers/PrerequisiteParameters.java @@ -25,7 +25,6 @@ import com.google.devtools.build.lib.packages.Aspect; import com.google.devtools.build.lib.packages.ConfiguredAttributeMapper; import com.google.devtools.build.lib.packages.Rule; -import com.google.devtools.build.lib.packages.Target; import com.google.devtools.build.lib.skyframe.BuildConfigurationKey; import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey; import javax.annotation.Nullable; @@ -34,7 +33,7 @@ /** Common parameters for computing prerequisites. */ public final class PrerequisiteParameters { private final ConfiguredTargetKey configuredTargetKey; - private final Target target; + @Nullable private final Rule associatedRule; private final ImmutableList aspects; private final StarlarkTransitionCache transitionCache; @@ -45,14 +44,14 @@ public final class PrerequisiteParameters { public PrerequisiteParameters( ConfiguredTargetKey configuredTargetKey, - Target target, + @Nullable Rule associatedRule, Iterable aspects, StarlarkTransitionCache transitionCache, @Nullable ToolchainCollection toolchainContexts, @Nullable ConfiguredAttributeMapper attributeMap, TransitiveDependencyState transitiveState) { this.configuredTargetKey = configuredTargetKey; - this.target = target; + this.associatedRule = associatedRule; this.aspects = ImmutableList.copyOf(aspects); this.transitionCache = transitionCache; this.toolchainContexts = toolchainContexts; @@ -64,13 +63,9 @@ public Label label() { return configuredTargetKey.getLabel(); } - public Target target() { - return target; - } - @Nullable public Rule associatedRule() { - return target.getAssociatedRule(); + return associatedRule; } @Nullable @@ -96,8 +91,12 @@ public ConfiguredAttributeMapper attributeMap() { return attributeMap; } + @Nullable public Location location() { - return target.getLocation(); + if (associatedRule == null) { + return null; + } + return associatedRule.getLocation(); } public BuildEventId eventId() { diff --git a/src/main/java/com/google/devtools/build/lib/analysis/producers/PrerequisitesProducer.java b/src/main/java/com/google/devtools/build/lib/analysis/producers/PrerequisitesProducer.java index 43a286440cb2d9..295eedb29706f3 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/producers/PrerequisitesProducer.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/producers/PrerequisitesProducer.java @@ -15,7 +15,6 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.devtools.build.lib.analysis.AspectResolutionHelpers.computeAspectCollection; -import static com.google.devtools.build.lib.analysis.producers.AttributeConfiguration.Kind.VISIBILITY; import static com.google.devtools.build.lib.skyframe.ConfiguredTargetAndData.SPLIT_DEP_ORDERING; import static java.util.Arrays.sort; @@ -23,11 +22,9 @@ import com.google.devtools.build.lib.analysis.AspectCollection; import com.google.devtools.build.lib.analysis.DuplicateException; import com.google.devtools.build.lib.analysis.InconsistentAspectOrderException; -import com.google.devtools.build.lib.analysis.InconsistentNullConfigException; import com.google.devtools.build.lib.analysis.InvalidVisibilityDependencyException; import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; import com.google.devtools.build.lib.analysis.config.DependencyEvaluationException; -import com.google.devtools.build.lib.analysis.configuredtargets.PackageGroupConfiguredTarget; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.devtools.build.lib.packages.Aspect; @@ -104,7 +101,6 @@ interface ResultSink { public StateMachine step(Tasks tasks, ExtendedEventHandler listener) { switch (configuration.kind()) { case VISIBILITY: - case NULL_CONFIGURATION: tasks.enqueue( new ConfiguredTargetAndDataProducer( getPrerequisiteKey(/* configurationKey= */ null), @@ -145,18 +141,9 @@ public void acceptConfiguredTargetAndData(ConfiguredTargetAndData value, int ind } @Override - public void acceptConfiguredTargetAndDataError(InconsistentNullConfigException error) { + public void acceptConfiguredTargetAndDataError(InvalidVisibilityDependencyException error) { hasError = true; - if (configuration.kind() == VISIBILITY) { - // The target was configurable, but used as a visibility dependency. This is invalid because - // only `PackageGroup`s are accepted as visibility dependencies and those are not - // configurable. Propagates the exception with more precise information. - sink.acceptPrerequisitesError(new InvalidVisibilityDependencyException(label)); - return; - } - // `configuration.kind()` was `NULL_CONFIGURATION`. This is only used when the target is in the - // same package as the parent and not configurable so this should never happen. - throw new IllegalStateException(error); + sink.acceptPrerequisitesError(error); } @Override @@ -170,15 +157,6 @@ private StateMachine computeConfiguredAspects(Tasks tasks, ExtendedEventHandler return DONE; } - if (configuration.kind() == VISIBILITY) { - // Verifies that the dependency is a `package_group`. The value is always at index 0 because - // the `VISIBILITY` configuration is always unary. - if (!(configuredTargets[0].getConfiguredTarget() instanceof PackageGroupConfiguredTarget)) { - sink.acceptPrerequisitesError(new InvalidVisibilityDependencyException(label)); - return DONE; - } - } - cleanupValues(); AspectCollection aspects; diff --git a/src/main/java/com/google/devtools/build/lib/analysis/producers/TargetAndConfigurationProducer.java b/src/main/java/com/google/devtools/build/lib/analysis/producers/TargetAndConfigurationProducer.java index b64d5fa7c0a507..c8fdc99f0de7c1 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/producers/TargetAndConfigurationProducer.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/producers/TargetAndConfigurationProducer.java @@ -22,6 +22,7 @@ import com.google.devtools.build.lib.actions.ActionLookupKey; import com.google.devtools.build.lib.analysis.ConfiguredTargetValue; import com.google.devtools.build.lib.analysis.InconsistentNullConfigException; +import com.google.devtools.build.lib.analysis.InvalidVisibilityDependencyException; import com.google.devtools.build.lib.analysis.TargetAndConfiguration; import com.google.devtools.build.lib.analysis.TransitiveDependencyState; import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; @@ -34,6 +35,7 @@ import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.devtools.build.lib.packages.NoSuchPackageException; import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.PackageGroup; import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.packages.RuleTransitionData; import com.google.devtools.build.lib.packages.Target; @@ -52,8 +54,13 @@ * Computes the target and configuration for a configured target key. * *

      If the key has a configuration and the target is configurable, attempts to apply a rule side - * transition. If the configuration changes, delegates to a target with the new configuration. If - * the target is not configurable, directly delegates to the null configuration. + * transition. If the target is not configurable, directly transitions to the null configuration. If + * the resulting configuration already has an owner, delegates to the owner instead of recomputing + * the configured target. + * + *

      If the key does not have a configuration, it was requested as a visibility dependency. + * Verifies that the {@link Target} is a {@link PackageGroup}, throwing {@link + * InvalidVisibilityDependencyException} if that's not the case. */ public final class TargetAndConfigurationProducer implements StateMachine, Consumer, TargetProducer.ResultSink { @@ -72,6 +79,7 @@ public abstract static class TargetAndConfigurationError { /** Tags the error type. */ public enum Kind { CONFIGURED_VALUE_CREATION, + INVALID_VISIBILITY_DEPENDENCY, INCONSISTENT_NULL_CONFIG } @@ -79,6 +87,8 @@ public enum Kind { public abstract ConfiguredValueCreationException configuredValueCreation(); + public abstract InvalidVisibilityDependencyException invalidVisibilityDependency(); + public abstract InconsistentNullConfigException inconsistentNullConfig(); private static TargetAndConfigurationError of(ConfiguredValueCreationException e) { @@ -86,6 +96,15 @@ private static TargetAndConfigurationError of(ConfiguredValueCreationException e .configuredValueCreation(e); } + // TODO(b/261521010): enable this error once Rule transitions are removed from dependency + // resolution. + // private static TargetAndConfigurationError of(InvalidVisibilityDependencyException e) { + // return AutoOneOf_TargetAndConfigurationProducer_TargetAndConfigurationError + // .invalidVisibilityDependency(e); + // } + + // TODO(b/261521010): delete this error once Rule transitions are removed from dependency + // resolution. private static TargetAndConfigurationError of(InconsistentNullConfigException e) { return AutoOneOf_TargetAndConfigurationProducer_TargetAndConfigurationError .inconsistentNullConfig(e); @@ -154,29 +173,41 @@ private StateMachine determineConfiguration(Tasks tasks, ExtendedEventHandler li if (configurationKey == null) { if (target.isConfigurable()) { // We somehow ended up in a target that requires a non-null configuration but with a key - // that doesn't have a configuration. This is always an error, but we need to bubble this - // up to the parent to provide more context. + // that doesn't have a configuration. This is always an error, but we need to analyze the + // dependencies of the latter target to realize that. Short-circuit the evaluation to avoid + // doing useless work and running code with a null configuration that's not prepared for it. sink.acceptTargetAndConfigurationError( TargetAndConfigurationError.of(new InconsistentNullConfigException())); return DONE; } + // TODO(b/261521010): after removing the rule transition from dependency resolution, the logic + // here changes. + // + // A null configuration key will only be used for visibility dependencies so when that's + // true, a check that the target is a PackageGroup will be performed, throwing + // InvalidVisibilityDependencyException on failure. + // + // The ConfiguredTargetKey cannot fan-in in this case. sink.acceptTargetAndConfiguration( new TargetAndConfiguration(target, /* configuration= */ null), preRuleTransitionKey); return DONE; } + // This may happen for top-level ConfiguredTargets. + // + // TODO(b/261521010): this may also happen for targets that are not top-level after removing + // rule transitions from dependency resolution. Update this comment. if (!target.isConfigurable()) { - // If target is not configurable, but requested with a configuration. Delegates to a key with - // the null configuration. This is expected to be uncommon. The common case of a - // non-configurable target is an input file, but those are usually package local and requested - // correctly with the null configuration. - delegateTo( - tasks, - ConfiguredTargetKey.builder() - .setLabel(preRuleTransitionKey.getLabel()) - .setExecutionPlatformLabel(preRuleTransitionKey.getExecutionPlatformLabel()) - .build() - .toKey()); + var nullConfiguredTargetKey = + ConfiguredTargetKey.builder().setDelegate(preRuleTransitionKey).build(); + ActionLookupKey delegate = nullConfiguredTargetKey.toKey(); + if (!delegate.equals(preRuleTransitionKey)) { + // Delegates to the key that already owns the null configuration. + delegateTo(tasks, delegate); + return DONE; + } + sink.acceptTargetAndConfiguration( + new TargetAndConfiguration(target, /* configuration= */ null), nullConfiguredTargetKey); return DONE; } @@ -236,15 +267,17 @@ private StateMachine processTransitionedKey(Tasks tasks, ExtendedEventHandler li } if (!configurationKey.equals(preRuleTransitionKey.getConfigurationKey())) { - delegateTo( - tasks, + fullKey = ConfiguredTargetKey.builder() - .setLabel(preRuleTransitionKey.getLabel()) - .setExecutionPlatformLabel(preRuleTransitionKey.getExecutionPlatformLabel()) + .setDelegate(preRuleTransitionKey) .setConfigurationKey(configurationKey) - .build() - .toKey()); - return DONE; + .build(); + ActionLookupKey delegate = fullKey.toKey(); + if (!delegate.equals(preRuleTransitionKey)) { + // Delegates to the key that already owns this configuration. + delegateTo(tasks, delegate); + return DONE; + } } else { fullKey = preRuleTransitionKey; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/BUILD index 765ded21504b72..b2d6686c088c40 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/BUILD @@ -181,6 +181,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/skyframe:skymeld_module", "//src/main/java/com/google/devtools/build/lib/standalone", "//src/main/java/com/google/devtools/build/lib/starlarkdebug/module", + "//src/main/java/com/google/devtools/build/lib/vfs/bazel", "//src/main/java/com/google/devtools/build/lib/worker:worker_module", "//third_party:guava", ], diff --git a/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java b/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java index 7a650366e58c06..5d12a774b4a2e3 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java @@ -28,7 +28,6 @@ */ public final class Bazel { private static final String BUILD_DATA_PROPERTIES = "/build-data.properties"; - /** * The list of modules to load. Note that the order is important: In case multiple modules provide * strategies for the same things, the last module wins and its strategy becomes the default. @@ -70,6 +69,7 @@ public final class Bazel { com.google.devtools.build.lib.standalone.StandaloneModule.class, com.google.devtools.build.lib.sandbox.SandboxModule.class, com.google.devtools.build.lib.runtime.BuildSummaryStatsModule.class, + com.google.devtools.build.lib.vfs.bazel.BazelHashFunctions.class, com.google.devtools.build.lib.dynamic.DynamicExecutionModule.class, com.google.devtools.build.lib.bazel.rules.BazelRulesModule.class, com.google.devtools.build.lib.bazel.rules.BazelStrategyModule.class, diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 82cb4a3f754dd6..1bbcd0110f90b5 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.RuleDefinition; @@ -48,6 +49,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.RepoSpec; import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionEvalFunction; import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionUsagesFunction; +import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsUtil; import com.google.devtools.build.lib.bazel.commands.FetchCommand; import com.google.devtools.build.lib.bazel.commands.ModqueryCommand; import com.google.devtools.build.lib.bazel.commands.SyncCommand; @@ -156,7 +158,9 @@ public class BazelRepositoryModule extends BlazeModule { private LockfileMode bazelLockfileMode = LockfileMode.OFF; private List allowedYankedVersions = ImmutableList.of(); private SingleExtensionEvalFunction singleExtensionEvalFunction; - private final ExecutorService repoFetchingWorkerThreadPool = Executors.newFixedThreadPool(100); + private final ExecutorService repoFetchingWorkerThreadPool = + Executors.newFixedThreadPool( + 100, new ThreadFactoryBuilder().setNameFormat("repo-fetching-worker-%d").build()); @Nullable private CredentialModule credentialModule; @@ -590,7 +594,7 @@ public ImmutableList getPrecomputedValues() { BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, bazelCompatibilityMode), PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, bazelLockfileMode), PrecomputedValue.injected( - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, allowedYankedVersions)); + YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, allowedYankedVersions)); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index b04cd1d270c42e..37b23eb6a56616 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -176,6 +176,7 @@ java_library( "SingleExtensionUsagesFunction.java", "StarlarkBazelModule.java", "TypeCheckedTag.java", + "YankedVersionsUtil.java", ], deps = [ ":common", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java index 94ea47b7fdb4d8..9e3ed795734e92 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java @@ -173,7 +173,7 @@ static BzlmodFlagsAndEnvVars getFlagsAndEnvVars(Environment env) throws Interrup (ClientEnvironmentValue) env.getValue( ClientEnvironmentFunction.key( - BazelModuleResolutionFunction.BZLMOD_ALLOWED_YANKED_VERSIONS_ENV)); + YankedVersionsUtil.BZLMOD_ALLOWED_YANKED_VERSIONS_ENV)); if (allowedYankedVersionsFromEnv == null) { return null; } @@ -185,7 +185,7 @@ static BzlmodFlagsAndEnvVars getFlagsAndEnvVars(Environment env) throws Interrup toImmutableMap(e -> e.getKey(), e -> ((LocalPathOverride) e.getValue()).getPath())); ImmutableList yankedVersions = - ImmutableList.copyOf(BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.get(env)); + ImmutableList.copyOf(YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.get(env)); Boolean ignoreDevDeps = ModuleFileFunction.IGNORE_DEV_DEPS.get(env); String compatabilityMode = BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.get(env).name(); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java index aebce5ce7ba5d5..2b84a14500fb24 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java @@ -15,27 +15,21 @@ package com.google.devtools.build.lib.bazel.bzlmod; -import com.google.common.base.Splitter; import com.google.common.base.Strings; import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.devtools.build.lib.analysis.BlazeVersionInfo; import com.google.devtools.build.lib.bazel.BazelVersion; import com.google.devtools.build.lib.bazel.bzlmod.InterimModule.DepSpec; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue; -import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; -import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; -import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction; -import com.google.devtools.build.lib.skyframe.ClientEnvironmentValue; import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionException; @@ -43,10 +37,8 @@ import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import java.io.IOException; -import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import javax.annotation.Nullable; /** @@ -59,22 +51,11 @@ public class BazelModuleResolutionFunction implements SkyFunction { new Precomputed<>("check_direct_dependency"); public static final Precomputed BAZEL_COMPATIBILITY_MODE = new Precomputed<>("bazel_compatibility_mode"); - public static final Precomputed> ALLOWED_YANKED_VERSIONS = - new Precomputed<>("allowed_yanked_versions"); - - public static final String BZLMOD_ALLOWED_YANKED_VERSIONS_ENV = "BZLMOD_ALLOW_YANKED_VERSIONS"; @Override @Nullable public SkyValue compute(SkyKey skyKey, Environment env) throws BazelModuleResolutionFunctionException, InterruptedException { - - ClientEnvironmentValue allowedYankedVersionsFromEnv = - (ClientEnvironmentValue) - env.getValue(ClientEnvironmentFunction.key(BZLMOD_ALLOWED_YANKED_VERSIONS_ENV)); - if (allowedYankedVersionsFromEnv == null) { - return null; - } RootModuleFileValue root = (RootModuleFileValue) env.getValue(ModuleFileValue.KEY_FOR_ROOT_MODULE); if (root == null) { @@ -104,12 +85,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) Objects.requireNonNull(BAZEL_COMPATIBILITY_MODE.get(env)), env.getListener()); - verifyYankedVersions( - resolvedDepGraph, - parseYankedVersions( - allowedYankedVersionsFromEnv.getValue(), - Objects.requireNonNull(ALLOWED_YANKED_VERSIONS.get(env))), - env.getListener()); + checkNoYankedVersions(resolvedDepGraph); ImmutableMap finalDepGraph = computeFinalDepGraph(resolvedDepGraph, root.getOverrides(), env.getListener()); @@ -194,125 +170,10 @@ public static void checkBazelCompatibility( } } - /** - * Parse a set of allowed yanked version from command line flag (--allowed_yanked_versions) and - * environment variable (ALLOWED_YANKED_VERSIONS). If `all` is specified, return Optional.empty(); - * otherwise returns the set of parsed modulel key. - */ - private Optional> parseYankedVersions( - String allowedYankedVersionsFromEnv, List allowedYankedVersionsFromFlag) - throws BazelModuleResolutionFunctionException { - ImmutableSet.Builder allowedYankedVersionBuilder = new ImmutableSet.Builder<>(); - if (allowedYankedVersionsFromEnv != null) { - if (parseModuleKeysFromString( - allowedYankedVersionsFromEnv, - allowedYankedVersionBuilder, - String.format( - "envirnoment variable %s=%s", - BZLMOD_ALLOWED_YANKED_VERSIONS_ENV, allowedYankedVersionsFromEnv))) { - return Optional.empty(); - } - } - for (String allowedYankedVersions : allowedYankedVersionsFromFlag) { - if (parseModuleKeysFromString( - allowedYankedVersions, - allowedYankedVersionBuilder, - String.format("command line flag --allow_yanked_versions=%s", allowedYankedVersions))) { - return Optional.empty(); - } - } - return Optional.of(allowedYankedVersionBuilder.build()); - } - - /** - * Parse of a comma-separated list of module version(s) of the form '@' or - * 'all' from the string. Returns true if 'all' is present, otherwise returns false. - */ - private boolean parseModuleKeysFromString( - String input, ImmutableSet.Builder allowedYankedVersionBuilder, String context) + private static void checkNoYankedVersions(ImmutableMap depGraph) throws BazelModuleResolutionFunctionException { - ImmutableList moduleStrs = ImmutableList.copyOf(Splitter.on(',').split(input)); - - for (String moduleStr : moduleStrs) { - if (moduleStr.equals("all")) { - return true; - } - - if (moduleStr.isEmpty()) { - continue; - } - - String[] pieces = moduleStr.split("@", 2); - - if (pieces.length != 2) { - throw new BazelModuleResolutionFunctionException( - ExternalDepsException.withMessage( - Code.VERSION_RESOLUTION_ERROR, - "Parsing %s failed, module versions must be of the form '@'", - context), - Transience.PERSISTENT); - } - - if (!RepositoryName.VALID_MODULE_NAME.matcher(pieces[0]).matches()) { - throw new BazelModuleResolutionFunctionException( - ExternalDepsException.withMessage( - Code.VERSION_RESOLUTION_ERROR, - "Parsing %s failed, invalid module name '%s': valid names must 1) only contain" - + " lowercase letters (a-z), digits (0-9), dots (.), hyphens (-), and" - + " underscores (_); 2) begin with a lowercase letter; 3) end with a lowercase" - + " letter or digit.", - context, - pieces[0]), - Transience.PERSISTENT); - } - - Version version; - try { - version = Version.parse(pieces[1]); - } catch (ParseException e) { - throw new BazelModuleResolutionFunctionException( - ExternalDepsException.withCauseAndMessage( - Code.VERSION_RESOLUTION_ERROR, - e, - "Parsing %s failed, invalid version specified for module: %s", - context, - pieces[1]), - Transience.PERSISTENT); - } - - allowedYankedVersionBuilder.add(ModuleKey.create(pieces[0], version)); - } - return false; - } - - private static void verifyYankedVersions( - ImmutableMap depGraph, - Optional> allowedYankedVersions, - ExtendedEventHandler eventHandler) - throws BazelModuleResolutionFunctionException, InterruptedException { - // Check whether all resolved modules are either not yanked or allowed. Modules with a - // NonRegistryOverride are ignored as their metadata is not available whatsoever. for (InterimModule m : depGraph.values()) { - if (m.getKey().equals(ModuleKey.ROOT) || m.getRegistry() == null) { - continue; - } - Optional> yankedVersions; - try { - yankedVersions = m.getRegistry().getYankedVersions(m.getKey().getName(), eventHandler); - } catch (IOException e) { - eventHandler.handle( - Event.warn( - String.format( - "Could not read metadata file for module %s: %s", m.getKey(), e.getMessage()))); - continue; - } - if (yankedVersions.isEmpty()) { - continue; - } - String yankedInfo = yankedVersions.get().get(m.getVersion()); - if (yankedInfo != null - && allowedYankedVersions.isPresent() - && !allowedYankedVersions.get().contains(m.getKey())) { + if (m.getYankedInfo().isPresent()) { throw new BazelModuleResolutionFunctionException( ExternalDepsException.withMessage( Code.VERSION_RESOLUTION_ERROR, @@ -322,7 +183,7 @@ private static void verifyYankedVersions( + "continue using this version, allow it using the --allow_yanked_versions " + "flag or the BZLMOD_ALLOW_YANKED_VERSIONS env variable.", m.getKey(), - yankedInfo), + m.getYankedInfo().get()), Transience.PERSISTENT); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/InterimModule.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/InterimModule.java index 06f5051a7c0c3a..b4a6a3d8fe50c7 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/InterimModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/InterimModule.java @@ -50,6 +50,9 @@ public abstract class InterimModule extends ModuleBase { /** List of bazel compatible versions that would run/fail this module */ public abstract ImmutableList getBazelCompatibility(); + /** The reason why this module was yanked or empty if it hasn't been yanked. */ + public abstract Optional getYankedInfo(); + /** The specification of a dependency. */ @AutoValue public abstract static class DepSpec { @@ -102,7 +105,8 @@ public static Builder builder() { .setName("") .setVersion(Version.EMPTY) .setKey(ModuleKey.ROOT) - .setCompatibilityLevel(0); + .setCompatibilityLevel(0) + .setYankedInfo(Optional.empty()); } /** @@ -133,6 +137,9 @@ public abstract static class Builder { /** Optional; defaults to {@link #setName}. */ public abstract Builder setRepoName(String value); + /** Optional; defaults to {@link Optional#empty()}. */ + public abstract Builder setYankedInfo(Optional value); + public abstract Builder setBazelCompatibility(ImmutableList value); abstract ImmutableList.Builder bazelCompatibilityBuilder(); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java index cccc0d79b7c627..02e62f19f8d904 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java @@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.devtools.build.lib.actions.FileValue; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.NonRootModuleFileValue; @@ -28,6 +29,8 @@ import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.rules.repository.RepositoryDirectoryValue; import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; +import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction; +import com.google.devtools.build.lib.skyframe.ClientEnvironmentValue; import com.google.devtools.build.lib.skyframe.PrecomputedValue; import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed; import com.google.devtools.build.lib.util.Fingerprint; @@ -102,10 +105,29 @@ public SkyValue compute(SkyKey skyKey, Environment env) return computeForRootModule(starlarkSemantics, env); } + ClientEnvironmentValue allowedYankedVersionsFromEnv = + (ClientEnvironmentValue) + env.getValue( + ClientEnvironmentFunction.key( + YankedVersionsUtil.BZLMOD_ALLOWED_YANKED_VERSIONS_ENV)); + if (allowedYankedVersionsFromEnv == null) { + return null; + } + + Optional> allowedYankedVersions; + try { + allowedYankedVersions = + YankedVersionsUtil.parseAllowedYankedVersions( + allowedYankedVersionsFromEnv.getValue(), + Objects.requireNonNull(YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.get(env))); + } catch (ExternalDepsException e) { + throw new ModuleFileFunctionException(e, SkyFunctionException.Transience.PERSISTENT); + } + ModuleFileValue.Key moduleFileKey = (ModuleFileValue.Key) skyKey; ModuleKey moduleKey = moduleFileKey.getModuleKey(); GetModuleFileResult getModuleFileResult = - getModuleFile(moduleKey, moduleFileKey.getOverride(), env); + getModuleFile(moduleKey, moduleFileKey.getOverride(), allowedYankedVersions, env); if (getModuleFileResult == null) { return null; } @@ -119,6 +141,8 @@ public SkyValue compute(SkyKey skyKey, Environment env) moduleKey, // Dev dependencies should always be ignored if the current module isn't the root module /* ignoreDevDeps= */ true, + // We try to prevent most side effects of yanked modules, in particular print(). + /* printIsNoop= */ getModuleFileResult.yankedInfo != null, starlarkSemantics, env); @@ -139,6 +163,23 @@ public SkyValue compute(SkyKey skyKey, Environment env) module.getVersion()); } + if (getModuleFileResult.yankedInfo != null) { + // Yanked modules should not have observable side effects such as adding dependency + // requirements, so we drop those from the constructed module. We do have to preserve the + // compatibility level as it influences the set of versions the yanked version can be updated + // to during selection. + return NonRootModuleFileValue.create( + InterimModule.builder() + .setKey(module.getKey()) + .setName(module.getName()) + .setVersion(module.getVersion()) + .setCompatibilityLevel(module.getCompatibilityLevel()) + .setRegistry(module.getRegistry()) + .setYankedInfo(Optional.of(getModuleFileResult.yankedInfo)) + .build(), + moduleFileHash); + } + return NonRootModuleFileValue.create(module, moduleFileHash); } @@ -159,6 +200,7 @@ private SkyValue computeForRootModule(StarlarkSemantics starlarkSemantics, Envir /* registry= */ null, ModuleKey.ROOT, /* ignoreDevDeps= */ Objects.requireNonNull(IGNORE_DEV_DEPS.get(env)), + /* printIsNoop= */ false, starlarkSemantics, env); InterimModule module = moduleFileGlobals.buildModule(); @@ -205,6 +247,7 @@ private ModuleFileGlobals execModuleFile( @Nullable Registry registry, ModuleKey moduleKey, boolean ignoreDevDeps, + boolean printIsNoop, StarlarkSemantics starlarkSemantics, Environment env) throws ModuleFileFunctionException, InterruptedException { @@ -223,7 +266,11 @@ private ModuleFileGlobals execModuleFile( Program program = Program.compileFile(starlarkFile, predeclaredEnv); // TODO(wyv): check that `program` has no `def`, `if`, etc StarlarkThread thread = new StarlarkThread(mu, starlarkSemantics); - thread.setPrintHandler(Event.makeDebugPrintHandler(env.getListener())); + if (printIsNoop) { + thread.setPrintHandler((t, msg) -> {}); + } else { + thread.setPrintHandler(Event.makeDebugPrintHandler(env.getListener())); + } Starlark.execFileProgram(program, predeclaredEnv, thread); } catch (SyntaxError.Exception e) { Event.replayEventsOn(env.getListener(), e.errors()); @@ -237,13 +284,19 @@ private ModuleFileGlobals execModuleFile( private static class GetModuleFileResult { ModuleFile moduleFile; + // `yankedInfo` is non-null if and only if the module has been yanked and hasn't been + // allowlisted. + @Nullable String yankedInfo; // `registry` can be null if this module has a non-registry override. @Nullable Registry registry; } @Nullable private GetModuleFileResult getModuleFile( - ModuleKey key, @Nullable ModuleOverride override, Environment env) + ModuleKey key, + @Nullable ModuleOverride override, + Optional> allowedYankedVersions, + Environment env) throws ModuleFileFunctionException, InterruptedException { // If there is a non-registry override for this module, we need to fetch the corresponding repo // first and read the module file from there. @@ -303,6 +356,10 @@ private GetModuleFileResult getModuleFile( } result.moduleFile = moduleFile.get(); result.registry = registry; + result.yankedInfo = + YankedVersionsUtil.getYankedInfo( + registry, key, allowedYankedVersions, env.getListener()) + .orElse(null); return result; } catch (IOException e) { throw errorf( @@ -346,5 +403,9 @@ static final class ModuleFileFunctionException extends SkyFunctionException { ModuleFileFunctionException(ExternalDepsException cause) { super(cause, Transience.TRANSIENT); } + + ModuleFileFunctionException(ExternalDepsException cause, Transience transience) { + super(cause, transience); + } } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/YankedVersionsUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/YankedVersionsUtil.java new file mode 100644 index 00000000000000..b6b5018b151286 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/YankedVersionsUtil.java @@ -0,0 +1,157 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.cmdline.RepositoryName; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.ExtendedEventHandler; +import com.google.devtools.build.lib.server.FailureDetails; +import com.google.devtools.build.lib.skyframe.PrecomputedValue; +import java.io.IOException; +import java.util.List; +import java.util.Optional; + +/** Utility class to parse and evaluate yanked version specifications and exceptions. */ +public final class YankedVersionsUtil { + + public static final PrecomputedValue.Precomputed> ALLOWED_YANKED_VERSIONS = + new PrecomputedValue.Precomputed<>("allowed_yanked_versions"); + public static final String BZLMOD_ALLOWED_YANKED_VERSIONS_ENV = "BZLMOD_ALLOW_YANKED_VERSIONS"; + + /** + * Parse a set of allowed yanked version from command line flag (--allowed_yanked_versions) and + * environment variable (ALLOWED_YANKED_VERSIONS). If `all` is specified, return Optional.empty(); + * otherwise returns the set of parsed modulel key. + */ + static Optional> parseAllowedYankedVersions( + String allowedYankedVersionsFromEnv, List allowedYankedVersionsFromFlag) + throws ExternalDepsException { + ImmutableSet.Builder allowedYankedVersionBuilder = new ImmutableSet.Builder<>(); + if (allowedYankedVersionsFromEnv != null) { + if (parseModuleKeysFromString( + allowedYankedVersionsFromEnv, + allowedYankedVersionBuilder, + String.format( + "environment variable %s=%s", + BZLMOD_ALLOWED_YANKED_VERSIONS_ENV, allowedYankedVersionsFromEnv))) { + return Optional.empty(); + } + } + for (String allowedYankedVersions : allowedYankedVersionsFromFlag) { + if (parseModuleKeysFromString( + allowedYankedVersions, + allowedYankedVersionBuilder, + String.format("command line flag --allow_yanked_versions=%s", allowedYankedVersions))) { + return Optional.empty(); + } + } + return Optional.of(allowedYankedVersionBuilder.build()); + } + + /** + * Returns the reason for the given module being yanked, or {@code Optional.empty()} if the module + * is not yanked or explicitly allowed despite being yanked. + */ + static Optional getYankedInfo( + Registry registry, + ModuleKey key, + Optional> allowedYankedVersions, + ExtendedEventHandler eventHandler) + throws InterruptedException { + Optional> yankedVersions; + try { + yankedVersions = registry.getYankedVersions(key.getName(), eventHandler); + } catch (IOException e) { + eventHandler.handle( + Event.warn( + String.format( + "Could not read metadata file for module %s: %s", key, e.getMessage()))); + // This is failing open: If we can't read the metadata file, we allow yanked modules to be + // fetched. + return Optional.empty(); + } + if (yankedVersions.isEmpty()) { + return Optional.empty(); + } + String yankedInfo = yankedVersions.get().get(key.getVersion()); + if (yankedInfo != null + && allowedYankedVersions.isPresent() + && !allowedYankedVersions.get().contains(key)) { + return Optional.of(yankedInfo); + } else { + return Optional.empty(); + } + } + + /** + * Parse of a comma-separated list of module version(s) of the form '@' or + * 'all' from the string. Returns true if 'all' is present, otherwise returns false. + */ + private static boolean parseModuleKeysFromString( + String input, ImmutableSet.Builder allowedYankedVersionBuilder, String context) + throws ExternalDepsException { + ImmutableList moduleStrs = ImmutableList.copyOf(Splitter.on(',').split(input)); + + for (String moduleStr : moduleStrs) { + if (moduleStr.equals("all")) { + return true; + } + + if (moduleStr.isEmpty()) { + continue; + } + + String[] pieces = moduleStr.split("@", 2); + + if (pieces.length != 2) { + throw ExternalDepsException.withMessage( + FailureDetails.ExternalDeps.Code.VERSION_RESOLUTION_ERROR, + "Parsing %s failed, module versions must be of the form '@'", + context); + } + + if (!RepositoryName.VALID_MODULE_NAME.matcher(pieces[0]).matches()) { + throw ExternalDepsException.withMessage( + FailureDetails.ExternalDeps.Code.VERSION_RESOLUTION_ERROR, + "Parsing %s failed, invalid module name '%s': valid names must 1) only contain" + + " lowercase letters (a-z), digits (0-9), dots (.), hyphens (-), and" + + " underscores (_); 2) begin with a lowercase letter; 3) end with a lowercase" + + " letter or digit.", + context, + pieces[0]); + } + + Version version; + try { + version = Version.parse(pieces[1]); + } catch (Version.ParseException e) { + throw ExternalDepsException.withCauseAndMessage( + FailureDetails.ExternalDeps.Code.VERSION_RESOLUTION_ERROR, + e, + "Parsing %s failed, invalid version specified for module: %s", + context, + pieces[1]); + } + + allowedYankedVersionBuilder.add(ModuleKey.create(pieces[0], version)); + } + return false; + } + + private YankedVersionsUtil() {} +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java index 5a4112308ba75f..e494a37d720f23 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRulesModule.java @@ -386,6 +386,23 @@ public static class BuildGraveyardOptions extends OptionsBase { metadataTags = {OptionMetadataTag.EXPERIMENTAL}, help = "Deprecated no-op.") public boolean enableBzlDocDump; + + // TODO(b/274595070): Remove this option. + @Option( + name = "experimental_parallel_aquery_output", + defaultValue = "true", + documentationCategory = OptionDocumentationCategory.QUERY, + effectTags = {OptionEffectTag.UNKNOWN}, + help = "No-op.") + public boolean parallelAqueryOutput; + + @Option( + name = "experimental_show_artifacts", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, + help = "Deprecated no-op.") + public boolean showArtifacts; } /** This is where deprecated Bazel-specific options only used by the build command go to die. */ diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/CcRules.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/CcRules.java index 021c560c3c12bd..f7b562ee9cd27c 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/CcRules.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/CcRules.java @@ -43,7 +43,6 @@ import com.google.devtools.build.lib.rules.cpp.DebugPackageProvider; import com.google.devtools.build.lib.rules.cpp.FdoPrefetchHintsRule; import com.google.devtools.build.lib.rules.cpp.FdoProfileRule; -import com.google.devtools.build.lib.rules.cpp.GraphNodeAspect; import com.google.devtools.build.lib.rules.cpp.PropellerOptimizeRule; import com.google.devtools.build.lib.rules.platform.PlatformRules; import com.google.devtools.build.lib.starlarkbuildapi.cpp.CcBootstrap; @@ -63,7 +62,6 @@ private CcRules() { @Override public void init(ConfiguredRuleClassProvider.Builder builder) { - GraphNodeAspect graphNodeAspect = new GraphNodeAspect(); BazelCcModule bazelCcModule = new BazelCcModule(); // TODO(gnish): This is only required for cc_toolchain_suite rule, // because it does not have AppleConfiguration fragment. @@ -76,7 +74,6 @@ public void init(ConfiguredRuleClassProvider.Builder builder) { builder.addBzlToplevel("cc_proto_aspect", Starlark.NONE); builder.addBuildInfoFactory(new CppBuildInfo()); - builder.addNativeAspectClass(graphNodeAspect); builder.addRuleDefinition(new CcToolchainRule()); builder.addRuleDefinition(new CcToolchainSuiteRule()); builder.addRuleDefinition(new CcToolchainAliasRule()); @@ -87,7 +84,7 @@ public void init(ConfiguredRuleClassProvider.Builder builder) { builder.addRuleDefinition(new BazelCppRuleClasses.CcDeclRule()); builder.addRuleDefinition(new BazelCppRuleClasses.CcBaseRule()); builder.addRuleDefinition(new BazelCppRuleClasses.CcRule()); - builder.addRuleDefinition(new BazelCppRuleClasses.CcBinaryBaseRule(graphNodeAspect)); + builder.addRuleDefinition(new BazelCppRuleClasses.CcBinaryBaseRule()); builder.addRuleDefinition(new BazelCcBinaryRule()); builder.addRuleDefinition(new CcSharedLibraryRule()); builder.addRuleDefinition(new BazelCcTestRule()); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/J2ObjcRules.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/J2ObjcRules.java index e953f68772203b..23b46fbbed7052 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/J2ObjcRules.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/J2ObjcRules.java @@ -16,10 +16,8 @@ import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider.RuleSet; -import com.google.devtools.build.lib.bazel.rules.cpp.BazelCppSemantics; import com.google.devtools.build.lib.bazel.rules.objc.BazelJ2ObjcLibraryRule; import com.google.devtools.build.lib.rules.core.CoreRules; -import com.google.devtools.build.lib.rules.objc.J2ObjcAspect; import com.google.devtools.build.lib.rules.objc.J2ObjcLibraryBaseRule; /** @@ -34,10 +32,7 @@ private J2ObjcRules() { @Override public void init(ConfiguredRuleClassProvider.Builder builder) { - J2ObjcAspect j2ObjcAspect = new J2ObjcAspect(builder, BazelCppSemantics.OBJC); - - builder.addNativeAspectClass(j2ObjcAspect); - builder.addRuleDefinition(new J2ObjcLibraryBaseRule(j2ObjcAspect)); + builder.addRuleDefinition(new J2ObjcLibraryBaseRule()); builder.addRuleDefinition(new BazelJ2ObjcLibraryRule()); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BUILD index cdfa3447fded8d..b46e763878c654 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BUILD @@ -35,7 +35,6 @@ java_library( "//src/main/java/com/google/devtools/build/lib/starlarkbuildapi/cpp", "//src/main/java/com/google/devtools/build/lib/util:filetype", "//src/main/java/com/google/devtools/build/lib/util:os", - "//src/main/java/net/starlark/java/eval", "//third_party:guava", "//third_party:jsr305", ], diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCppRuleClasses.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCppRuleClasses.java index 71f9612e9ff7ae..b638612a3c7d0f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCppRuleClasses.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BazelCppRuleClasses.java @@ -57,7 +57,6 @@ import com.google.devtools.build.lib.rules.cpp.CppFileTypes; import com.google.devtools.build.lib.rules.cpp.CppRuleClasses; import com.google.devtools.build.lib.rules.cpp.CppRuleClasses.CcIncludeScanningRule; -import com.google.devtools.build.lib.rules.cpp.GraphNodeAspect; import com.google.devtools.build.lib.util.FileTypeSet; import javax.annotation.Nullable; @@ -485,12 +484,6 @@ public Metadata getMetadata() { /** Helper rule class. */ public static final class CcBinaryBaseRule implements RuleDefinition { - private final GraphNodeAspect graphNodeAspect; - - public CcBinaryBaseRule(GraphNodeAspect graphNodeAspect) { - this.graphNodeAspect = graphNodeAspect; - } - @Override public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) { return builder @@ -511,8 +504,7 @@ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) .allowedRuleClasses(DEPS_ALLOWED_RULES) .allowedFileTypes(CppFileTypes.LINKER_SCRIPT) .skipAnalysisTimeFileTypeCheck() - .mandatoryProviders(StarlarkProviderIdentifier.forKey(CcInfo.PROVIDER.getKey())) - .aspect(graphNodeAspect, GraphNodeAspect.ASPECT_PARAMETERS)) + .mandatoryProviders(StarlarkProviderIdentifier.forKey(CcInfo.PROVIDER.getKey()))) .add( attr("dynamic_deps", LABEL_LIST) .allowedFileTypes(FileTypeSet.NO_FILE) @@ -541,8 +533,7 @@ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) attr("malloc", LABEL) .value(env.getToolsLabel("//tools/cpp:malloc")) .allowedFileTypes() - .allowedRuleClasses("cc_library") - .aspect(graphNodeAspect, GraphNodeAspect.ASPECT_PARAMETERS)) + .allowedRuleClasses("cc_library")) .add(attr(":default_malloc", LABEL).value(CppRuleClasses.DEFAULT_MALLOC)) /* Whether to encode build information into the binary. Possible values: diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinaryRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinaryRule.java index c89c619cf96689..87bd95df34ff60 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinaryRule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BazelJavaBinaryRule.java @@ -120,7 +120,9 @@ public Metadata getMetadata() {

      Builds a Java archive ("jar file"), plus a wrapper shell script with the same name as the rule. The wrapper shell script uses a classpath that includes, among other things, a jar file for each - library on which the binary depends. + library on which the binary depends. When running the wrapper shell script, any nonempty + JAVABIN environment variable will take precedence over the version specified via + Bazel's --java_runtime_version flag.

      The wrapper script accepts several unique flags. Refer to diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/objc/BazelJ2ObjcLibrary.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/objc/BazelJ2ObjcLibrary.java deleted file mode 100644 index d1b67993875699..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/objc/BazelJ2ObjcLibrary.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2021 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.bazel.rules.objc; - -import com.google.devtools.build.lib.bazel.rules.cpp.BazelCppSemantics; -import com.google.devtools.build.lib.rules.objc.J2ObjcLibrary; - -/** Factory class for the {@code j2objc_library} rule. */ -public class BazelJ2ObjcLibrary extends J2ObjcLibrary { - public BazelJ2ObjcLibrary() { - super(BazelCppSemantics.OBJC); - } -} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/objc/BazelJ2ObjcLibraryRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/objc/BazelJ2ObjcLibraryRule.java index 8f96ca75fd5a8e..01f0a73438f37b 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/objc/BazelJ2ObjcLibraryRule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/objc/BazelJ2ObjcLibraryRule.java @@ -14,6 +14,7 @@ package com.google.devtools.build.lib.bazel.rules.objc; +import com.google.devtools.build.lib.analysis.BaseRuleClasses; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; import com.google.devtools.build.lib.packages.RuleClass; @@ -34,7 +35,7 @@ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) public Metadata getMetadata() { return RuleDefinition.Metadata.builder() .name("j2objc_library") - .factoryClass(BazelJ2ObjcLibrary.class) + .factoryClass(BaseRuleClasses.EmptyRuleConfiguredTargetFactory.class) .ancestors(J2ObjcLibraryBaseRule.class, ObjcRuleClasses.CrosstoolRule.class) .build(); } diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/AqueryProcessor.java b/src/main/java/com/google/devtools/build/lib/buildtool/AqueryProcessor.java index d8095da4d90a98..f804dd30db86ca 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/AqueryProcessor.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/AqueryProcessor.java @@ -80,8 +80,7 @@ public BlazeCommandResult dumpActionGraphFromSkyframe(CommandEnvironment env) { ActionGraphProtoOutputFormatterCallback.constructAqueryOutputHandler( OutputType.fromString(aqueryOptions.outputFormat), queryRuntimeHelper.getOutputStreamForQueryOutput(), - printStream, - aqueryOptions.parallelAqueryOutput)) { + printStream)) { ActionGraphDump actionGraphDump = new ActionGraphDump( aqueryOptions.includeCommandline, diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java index 540f86c151ba7a..6485cb0a2c3615 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java @@ -203,18 +203,6 @@ public class BuildRequestOptions extends OptionsBase { + " under the threshold.") public int maxResultTargets; - @Option( - name = "experimental_show_artifacts", - defaultValue = "false", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, - help = - "Output a list of all top level artifacts produced by this build." - + "Use output format suitable for tool consumption. " - + "This flag is temporary and intended to facilitate Android Studio integration. " - + "This output format will likely change in the future or disappear completely.") - public boolean showArtifacts; - @Option( name = "announce", defaultValue = "false", diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildResultPrinter.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildResultPrinter.java index b42577ec016867..03f03d4b3757d5 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildResultPrinter.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildResultPrinter.java @@ -29,8 +29,6 @@ import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; import com.google.devtools.build.lib.analysis.configuredtargets.OutputFileConfiguredTarget; import com.google.devtools.build.lib.cmdline.Label; -import com.google.devtools.build.lib.collect.nestedset.NestedSet; -import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; import com.google.devtools.build.lib.exec.ExecutionOptions; import com.google.devtools.build.lib.runtime.BlazeRuntime; import com.google.devtools.build.lib.runtime.CommandEnvironment; @@ -360,41 +358,6 @@ private static String formatArtifactForShowResults( return " " + prettyPrinter.getPrettyPath(artifact.getPath().asFragment()); } - /** - * Prints a flat list of all artifacts built by the passed top-level targets. - * - *

      This corresponds to the --experimental_show_artifacts flag. - */ - void showArtifacts( - BuildRequest request, - Collection configuredTargets, - Collection aspects) { - - TopLevelArtifactContext context = request.getTopLevelArtifactContext(); - Collection targetsToPrint = filterTargetsToPrint(configuredTargets); - - NestedSetBuilder artifactsBuilder = NestedSetBuilder.stableOrder(); - targetsToPrint.forEach( - t -> - artifactsBuilder.addTransitive( - TopLevelArtifactHelper.getAllArtifactsToBuild(t, context).getImportantArtifacts())); - - aspects.forEach( - a -> - artifactsBuilder.addTransitive( - TopLevelArtifactHelper.getAllArtifactsToBuild(a, context).getImportantArtifacts())); - - OutErr outErr = request.getOutErr(); - outErr.printErrLn("Build artifacts:"); - - NestedSet artifacts = artifactsBuilder.build(); - for (Artifact artifact : artifacts.toList()) { - if (!artifact.isSourceArtifact()) { - outErr.printErrLn(">>>" + artifact.getPath()); - } - } - } - /** * Returns a list of configured targets that should participate in printing. * diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java index 506c1875343b7a..3a871c2901af85 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java @@ -576,16 +576,6 @@ void nonCatastrophicFinalizations( buildResultListener.getAnalyzedAspects()); } - try (SilentCloseable c = Profiler.instance().profile("Show artifacts")) { - if (request.getBuildOptions().showArtifacts) { - BuildResultPrinter buildResultPrinter = new BuildResultPrinter(env); - buildResultPrinter.showArtifacts( - request, - buildResultListener.getAnalyzedTargets(), - buildResultListener.getAnalyzedAspects().values()); - } - } - if (explanationHandler != null) { uninstallExplanationHandler(explanationHandler); try { diff --git a/src/main/java/com/google/devtools/build/lib/cmdline/Label.java b/src/main/java/com/google/devtools/build/lib/cmdline/Label.java index 0b8f66c94580e3..25e432e4b9346d 100644 --- a/src/main/java/com/google/devtools/build/lib/cmdline/Label.java +++ b/src/main/java/com/google/devtools/build/lib/cmdline/Label.java @@ -19,6 +19,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ComparisonChain; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Interner; import com.google.common.util.concurrent.Striped; import com.google.devtools.build.docgen.annot.DocCategory; import com.google.devtools.build.lib.actions.CommandLineItem; @@ -33,6 +34,7 @@ import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.skyframe.SkyFunctionName; import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.UsePooledLabelInterningFlag; import java.util.Arrays; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; @@ -84,10 +86,16 @@ public final class Label implements Comparable

      This is needed to support C++-related rule classes - * which accesses {@link #getDefaultHdrsCheck} from the still-under-construction - * package. - */ - private void setDefaultHdrsCheck(String defaultHdrsCheck) { - this.defaultHdrsCheck = defaultHdrsCheck; - } - /** * Returns the source root (a directory) beneath which this package's BUILD file was found, or * {@link Optional#empty} if this package was derived from a workspace file. @@ -643,11 +629,6 @@ public String getWorkspaceName() { return workspaceName; } - /** Returns the features specified in the package() declaration. */ - public FeatureSet getFeatures() { - return packageArgs.features(); - } - /** * Returns the target (a member of this package) whose name is "targetName". * First rules are searched, then output files, then input files. The target @@ -723,13 +704,6 @@ private String getAlternateTargetSuggestion(String targetName) { } } - /** - * Returns the default visibility for this package. - */ - public RuleVisibility getDefaultVisibility() { - return packageArgs.defaultVisibility(); - } - /** * How to enforce visibility on config_setting See * {@link ConfigSettingVisibilityPolicy} for details. @@ -738,63 +712,6 @@ public ConfigSettingVisibilityPolicy getConfigSettingVisibilityPolicy() { return configSettingVisibilityPolicy; } - /** - * Returns the default testonly value. - */ - public Boolean getDefaultTestOnly() { - return packageArgs.defaultTestOnly(); - } - - /** - * Returns the default deprecation value. - */ - public String getDefaultDeprecation() { - return packageArgs.defaultDeprecation(); - } - - /** Gets the default header checking mode. */ - public String getDefaultHdrsCheck() { - return defaultHdrsCheck != null ? defaultHdrsCheck : "strict"; - } - - /** - * Returns whether the default header checking mode has been set or it is the - * default value. - */ - public boolean isDefaultHdrsCheckSet() { - return defaultHdrsCheck != null; - } - - /** Gets the package metadata list for the default metadata declared by this package. */ - ImmutableList

      Unless otherwise specified, these are only used when the rule does not provide an explicit + * override value in the associated attribute. */ @AutoValue public abstract class PackageArgs { @@ -46,40 +49,61 @@ public abstract class PackageArgs { .setDefaultPackageMetadata(ImmutableList.of()) .build(); - /** See {@link Package#getDefaultVisibility()}. */ + /** The default visibility value for the package. */ @Nullable - abstract RuleVisibility defaultVisibility(); + public abstract RuleVisibility defaultVisibility(); - /** See {@link Package#getDefaultTestOnly()}. */ + /** The default testonly value for the package. */ @Nullable - abstract Boolean defaultTestOnly(); + public abstract Boolean defaultTestOnly(); - /** See {@link Package#getDefaultDeprecation()}. */ + /** The default deprecation value for the package. */ @Nullable - abstract String defaultDeprecation(); + public abstract String defaultDeprecation(); - /** See {@link Package#getFeatures()}. */ - abstract FeatureSet features(); + /** + * The default (generally C/C++) features value for the package. + * + *

      Note that this is actually additive with features set by a rule where the rule has priority + * for turning specific features on or off. + */ + public abstract FeatureSet features(); - /** See {@link Package#getDefaultLicense()}. */ + /** The default license value for the package. */ @Nullable - abstract License license(); + public abstract License license(); - /** See {@link Package#getDefaultDistribs()}. */ + /** The default distributions value for the package. */ @Nullable - abstract ImmutableSet distribs(); + public abstract ImmutableSet distribs(); - /** See {@link Package#getDefaultCompatibleWith()}. */ + /** The default {@link RuleClass#COMPATIBLE_ENVIRONMENT_ATTR} value for the package. */ @Nullable - abstract ImmutableSet

      TODO(b/128341904): Do cross module optimization once there is Starlark support. + */ @Override public LtoBackendArtifacts createLtoBackendArtifacts( StarlarkRuleContext starlarkRuleContext, String ltoOutputRootPrefixString, + String ltoObjRootPrefixString, Artifact bitcodeFile, FeatureConfigurationForStarlark featureConfigurationForStarlark, CcToolchainProvider ccToolchain, @@ -957,6 +965,7 @@ public LtoBackendArtifacts createLtoBackendArtifacts( isCalledFromStarlarkCcCommon(thread); RuleContext ruleContext = starlarkRuleContext.getRuleContext(); PathFragment ltoOutputRootPrefix = PathFragment.create(ltoOutputRootPrefixString); + PathFragment ltoObjRootPrefix = PathFragment.create(ltoObjRootPrefixString); LtoBackendArtifacts ltoBackendArtifacts; try { ltoBackendArtifacts = @@ -966,7 +975,9 @@ public LtoBackendArtifacts createLtoBackendArtifacts( ruleContext.getConfiguration().getOptions(), ruleContext.getConfiguration().getFragment(CppConfiguration.class), ltoOutputRootPrefix, + ltoObjRootPrefix, bitcodeFile, + /* allBitcodeFiles= */ null, starlarkRuleContext.actions().getActionConstructionContext(), ruleContext.getRepository(), ruleContext.getConfiguration(), @@ -2802,7 +2813,7 @@ public CcLinkingOutputs link( helper.addVariableExtension(new UserVariablesExtension(asDict(variablesExtension))); } if (convertFromNoneable(useShareableArtifactFactory, false)) { - helper.setLinkArtifactFactory(CppLinkActionBuilder.SHAREABLE_LINK_ARTIFACT_FACTORY); + helper.setLinkArtifactFactory(CppLinkAction.SHAREABLE_LINK_ARTIFACT_FACTORY); } CcCompilationOutputs compilationOutputs = convertFromNoneable(compilationOutputsObject, /* defaultValue= */ null); diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcStarlarkInternal.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcStarlarkInternal.java index 41763cb00ba067..d512c97ee472be 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcStarlarkInternal.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcStarlarkInternal.java @@ -343,7 +343,12 @@ public CcToolchainFeatures ccToolchainFeatures( parameters = {@Param(name = "ctx", positional = false, named = true)}) public boolean isPackageHeadersCheckingModeSetForStarlark( StarlarkRuleContext starlarkRuleContext) { - return starlarkRuleContext.getRuleContext().getRule().getPackage().isDefaultHdrsCheckSet(); + return starlarkRuleContext + .getRuleContext() + .getRule() + .getPackage() + .getPackageArgs() + .isDefaultHdrsCheckSet(); } @StarlarkMethod( @@ -351,7 +356,12 @@ public boolean isPackageHeadersCheckingModeSetForStarlark( documented = false, parameters = {@Param(name = "ctx", positional = false, named = true)}) public String getPackageHeadersCheckingModeForStarlark(StarlarkRuleContext starlarkRuleContext) { - return starlarkRuleContext.getRuleContext().getRule().getPackage().getDefaultHdrsCheck(); + return starlarkRuleContext + .getRuleContext() + .getRule() + .getPackage() + .getPackageArgs() + .getDefaultHdrsCheck(); } @StarlarkMethod( @@ -360,7 +370,12 @@ public String getPackageHeadersCheckingModeForStarlark(StarlarkRuleContext starl parameters = {@Param(name = "ctx", positional = false, named = true)}) public boolean isPackageHeadersCheckingModeSetForStarlarkAspect( StarlarkRuleContext starlarkRuleContext) { - return starlarkRuleContext.getRuleContext().getTarget().getPackage().isDefaultHdrsCheckSet(); + return starlarkRuleContext + .getRuleContext() + .getTarget() + .getPackage() + .getPackageArgs() + .isDefaultHdrsCheckSet(); } @StarlarkMethod( @@ -369,7 +384,12 @@ public boolean isPackageHeadersCheckingModeSetForStarlarkAspect( parameters = {@Param(name = "ctx", positional = false, named = true)}) public String getPackageHeadersCheckingModeForStarlarkAspect( StarlarkRuleContext starlarkRuleContext) { - return starlarkRuleContext.getRuleContext().getTarget().getPackage().getDefaultHdrsCheck(); + return starlarkRuleContext + .getRuleContext() + .getTarget() + .getPackage() + .getPackageArgs() + .getDefaultHdrsCheck(); } @StarlarkMethod( @@ -418,7 +438,9 @@ static class DefaultHdrsCheckBuiltinComputedDefault extends ComputedDefault implements NativeComputedDefaultApi { @Override public Object getDefault(AttributeMap rule) { - return rule.isPackageDefaultHdrsCheckSet() ? rule.getPackageDefaultHdrsCheck() : ""; + return rule.getPackageArgs().isDefaultHdrsCheckSet() + ? rule.getPackageArgs().getDefaultHdrsCheck() + : ""; } @Override diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java index 655a5e3192baba..3e43c5a62d0b0d 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java @@ -30,7 +30,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; import com.google.common.io.ByteStreams; import com.google.devtools.build.lib.actions.AbstractAction; import com.google.devtools.build.lib.actions.ActionEnvironment; @@ -124,8 +123,6 @@ public class CppCompileAction extends AbstractAction implements IncludeScannable private static final PathFragment BUILD_PATH_FRAGMENT = PathFragment.create("BUILD"); - private static final boolean VALIDATION_DEBUG_WARN = false; - @VisibleForTesting static final String CPP_COMPILE_MNEMONIC = "CppCompile"; @VisibleForTesting static final String OBJC_COMPILE_MNEMONIC = "ObjcCompile"; @@ -1058,29 +1055,6 @@ public void validateInclusions( errors.add(input.getExecPath().toString()); } } - if (VALIDATION_DEBUG_WARN) { - synchronized (System.err) { - if (errors.hasProblems()) { - if (errors.hasProblems()) { - System.err.println("ERROR: Include(s) were not in declared srcs:"); - } else { - System.err.println( - "INFO: Include(s) were OK for '" + getSourceFile() + "', declared srcs:"); - } - for (Artifact a : ccCompilationContext.getDeclaredIncludeSrcs().toList()) { - System.err.println(" '" + a.toDetailString() + "'"); - } - System.err.println(" or under loose headers dirs:"); - for (PathFragment f : Sets.newTreeSet(ccCompilationContext.getLooseHdrsDirs().toList())) { - System.err.println(" '" + f + "'"); - } - System.err.println(" with prefixes:"); - for (PathFragment dirpath : ccCompilationContext.getQuoteIncludeDirs()) { - System.err.println(" '" + dirpath + "'"); - } - } - } - } errors.assertProblemFree(this, getSourceFile()); } diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionTemplate.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionTemplate.java index 839a8f6312a623..64c7b12835cb5a 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionTemplate.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionTemplate.java @@ -17,6 +17,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.devtools.build.lib.actions.ActionAnalysisMetadata; import com.google.devtools.build.lib.actions.ActionExecutionContext; import com.google.devtools.build.lib.actions.ActionExecutionException; @@ -38,6 +39,7 @@ import com.google.devtools.build.lib.util.DetailedExitCode; import com.google.devtools.build.lib.util.Fingerprint; import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; import javax.annotation.Nullable; /** An {@link ActionTemplate} that expands into {@link CppCompileAction}s at execution time. */ @@ -48,6 +50,7 @@ public final class CppCompileActionTemplate extends ActionKeyCacher private final SpecialArtifact outputTreeArtifact; private final SpecialArtifact dotdTreeArtifact; private final SpecialArtifact diagnosticsTreeArtifact; + private final SpecialArtifact ltoIndexTreeArtifact; private final CcToolchainProvider toolchain; private final ImmutableList categories; private final ActionOwner actionOwner; @@ -61,6 +64,7 @@ public final class CppCompileActionTemplate extends ActionKeyCacher * @param outputTreeArtifact the TreeArtifact that contains compilation outputs. * @param dotdTreeArtifact the TreeArtifact that contains dotd files. * @param diagnosticsTreeArtifact the TreeArtifact that contains serialized diagnostics files. + * @param ltoIndexTreeArtifact the TreeArtifact that contains lto index files (minimized bitcode). * @param cppCompileActionBuilder An almost completely configured {@link CppCompileActionBuilder} * without the input and output files set. It is used as a template to instantiate expanded * {CppCompileAction}s. @@ -74,6 +78,7 @@ public final class CppCompileActionTemplate extends ActionKeyCacher SpecialArtifact outputTreeArtifact, SpecialArtifact dotdTreeArtifact, SpecialArtifact diagnosticsTreeArtifact, + SpecialArtifact ltoIndexTreeArtifact, CppCompileActionBuilder cppCompileActionBuilder, CcToolchainProvider toolchain, ImmutableList categories, @@ -82,6 +87,7 @@ public final class CppCompileActionTemplate extends ActionKeyCacher this.sourceTreeArtifact = sourceTreeArtifact; this.outputTreeArtifact = outputTreeArtifact; this.dotdTreeArtifact = dotdTreeArtifact; + this.ltoIndexTreeArtifact = ltoIndexTreeArtifact; this.diagnosticsTreeArtifact = diagnosticsTreeArtifact; this.toolchain = toolchain; this.categories = categories; @@ -146,12 +152,25 @@ public ImmutableList generateActionsForInputArtifacts( TreeFileArtifact.createTemplateExpansionOutput( diagnosticsTreeArtifact, outputName + ".dia", artifactOwner); } + + TreeFileArtifact ltoIndexFileArtifact = null; + if (ltoIndexTreeArtifact != null) { + PathFragment outputFilePathFragment = PathFragment.create(outputName); + PathFragment thinltofile = + FileSystemUtils.replaceExtension( + outputFilePathFragment, + Iterables.getOnlyElement(CppFileTypes.LTO_INDEXING_OBJECT_FILE.getExtensions())); + ltoIndexFileArtifact = + TreeFileArtifact.createTemplateExpansionOutput( + ltoIndexTreeArtifact, thinltofile, artifactOwner); + } expandedActions.add( createAction( inputTreeFileArtifact, outputTreeFileArtifact, dotdFileArtifact, diagnosticsFileArtifact, + ltoIndexFileArtifact, privateHeaders)); } @@ -201,13 +220,15 @@ private CppCompileAction createAction( TreeFileArtifact outputTreeFileArtifact, @Nullable Artifact dotdFileArtifact, @Nullable Artifact diagnosticsFileArtifact, + @Nullable Artifact ltoIndexFileArtifact, NestedSet privateHeaders) throws ActionExecutionException { CppCompileActionBuilder builder = new CppCompileActionBuilder(cppCompileActionBuilder) .setAdditionalPrunableHeaders(privateHeaders) .setSourceFile(sourceTreeFileArtifact) - .setOutputs(outputTreeFileArtifact, dotdFileArtifact, diagnosticsFileArtifact); + .setOutputs(outputTreeFileArtifact, dotdFileArtifact, diagnosticsFileArtifact) + .setLtoIndexingFile(ltoIndexFileArtifact); CcToolchainVariables.Builder buildVariables = CcToolchainVariables.builder(cppCompileActionBuilder.getVariables()); @@ -228,6 +249,12 @@ private CppCompileAction createAction( diagnosticsFileArtifact.getExecPathString()); } + if (ltoIndexFileArtifact != null) { + buildVariables.overrideStringVariable( + CompileBuildVariables.LTO_INDEXING_BITCODE_FILE.getVariableName(), + ltoIndexFileArtifact.getExecPathString()); + } + builder.setVariables(buildVariables.build()); try { @@ -316,10 +343,15 @@ public NestedSet getSchedulingDependencies() { @Override public ImmutableSet getOutputs() { - if (dotdTreeArtifact == null) { - return ImmutableSet.of(outputTreeArtifact); + ImmutableSet.Builder builder = ImmutableSet.builder(); + builder.add(outputTreeArtifact); + if (dotdTreeArtifact != null) { + builder.add(dotdTreeArtifact); + } + if (ltoIndexTreeArtifact != null) { + builder.add(ltoIndexTreeArtifact); } - return ImmutableSet.of(outputTreeArtifact, dotdTreeArtifact); + return builder.build(); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppFileTypes.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppFileTypes.java index c55cf90028708f..ea8ee704338e67 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppFileTypes.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppFileTypes.java @@ -239,6 +239,13 @@ public ImmutableList getExtensions() { // Minimized bitcode file emitted by the ThinLTO compile step and used just for LTO indexing. public static final FileType LTO_INDEXING_OBJECT_FILE = FileType.of(".indexing.o"); + // Imports file emitted by the ThinLTO indexing step and used for LTO backend action. + public static final FileType LTO_IMPORTS_FILE = FileType.of(".imports"); + + // Indexing analysis result file emitted by the ThinLTO indexing step and used for LTO backend + // action. + public static final FileType LTO_INDEXING_ANALYSIS_FILE = FileType.of(".thinlto.bc"); + // TODO(bazel-team): File types should not be read from this hard-coded list but should come from // the toolchain instead. See https://github.com/bazelbuild/bazel/issues/17117 public static final FileType SHARED_LIBRARY = diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppHelper.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppHelper.java index 7ce7419c27d4e4..de4f753f66770e 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppHelper.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppHelper.java @@ -51,6 +51,7 @@ import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType; import com.google.devtools.build.lib.server.FailureDetails.FailAction.Code; import com.google.devtools.build.lib.util.FileTypeSet; +import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.PathFragment; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -73,6 +74,9 @@ public class CppHelper { static final PathFragment DIA_FILES = PathFragment.create("_dia"); static final PathFragment PIC_DIA_FILES = PathFragment.create("_pic_dia"); + public static final PathFragment SHARED_NONLTO_BACKEND_ROOT_PREFIX = + PathFragment.create("shared.nonlto"); + // TODO(bazel-team): should this use Link.SHARED_LIBRARY_FILETYPES? public static final FileTypeSet SHARED_LIBRARY_FILETYPES = FileTypeSet.of(CppFileTypes.SHARED_LIBRARY, CppFileTypes.VERSIONED_SHARED_LIBRARY); @@ -264,6 +268,23 @@ private static PathFragment getDiagnosticsDirectory( ruleLabel, usePic ? PIC_DIA_FILES : DIA_FILES, siblingRepositoryLayout); } + /** + * Given the output file path, returns the directory where the results of thinlto indexing will be + * created: output_file.lto/ + */ + public static PathFragment getLtoOutputRootPrefix(PathFragment outputRootRelativePath) { + return FileSystemUtils.appendExtension(outputRootRelativePath, ".lto"); + } + + /** + * Given the lto output root directory path, returns the directory where thinlto native object + * files are created: output_file.lto-obj/ + */ + public static PathFragment getThinLtoNativeObjectDirectoryFromLtoOutputRoot( + PathFragment ltoOutputRootRelativePath) { + return FileSystemUtils.appendExtension(ltoOutputRootRelativePath, "-obj"); + } + public static Artifact getLinkedArtifact( Label label, ActionConstructionContext actionConstructionContext, diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkAction.java index 6b22b982e0fe12..9c4a34372341d4 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkAction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkAction.java @@ -31,6 +31,7 @@ import com.google.devtools.build.lib.actions.ActionResult; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander; +import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact; import com.google.devtools.build.lib.actions.CommandAction; import com.google.devtools.build.lib.actions.CommandLine; import com.google.devtools.build.lib.actions.CommandLineExpansionException; @@ -88,6 +89,13 @@ Artifact create( RepositoryName repositoryName, BuildConfigurationValue configuration, PathFragment rootRelativePath); + + /** Create a tree artifact at the specified root-relative path in the bin directory. */ + SpecialArtifact createTreeArtifact( + ActionConstructionContext actionConstructionContext, + RepositoryName repositoryName, + BuildConfigurationValue configuration, + PathFragment rootRelativePath); } /** @@ -105,6 +113,49 @@ public Artifact create( return actionConstructionContext.getDerivedArtifact( rootRelativePath, configuration.getBinDirectory(repositoryName)); } + + @Override + public SpecialArtifact createTreeArtifact( + ActionConstructionContext actionConstructionContext, + RepositoryName repositoryName, + BuildConfigurationValue configuration, + PathFragment rootRelativePath) { + return actionConstructionContext.getTreeArtifact( + rootRelativePath, configuration.getBinDirectory(repositoryName)); + } + }; + + /** + * An implementation of {@link LinkArtifactFactory} that can create artifacts anywhere. + * + *

      Necessary when the LTO backend actions of libraries should be shareable, and thus cannot be + * under the package directory. + * + *

      Necessary because the actions of nativedeps libraries should be shareable, and thus cannot + * be under the package directory. + */ + public static final LinkArtifactFactory SHAREABLE_LINK_ARTIFACT_FACTORY = + new LinkArtifactFactory() { + @Override + public Artifact create( + ActionConstructionContext actionConstructionContext, + RepositoryName repositoryName, + BuildConfigurationValue configuration, + PathFragment rootRelativePath) { + return actionConstructionContext.getShareableArtifact( + rootRelativePath, configuration.getBinDirectory(repositoryName)); + } + + @Override + public SpecialArtifact createTreeArtifact( + ActionConstructionContext actionConstructionContext, + RepositoryName repositoryName, + BuildConfigurationValue configuration, + PathFragment rootRelativePath) { + return actionConstructionContext + .getAnalysisEnvironment() + .getTreeArtifact(rootRelativePath, configuration.getBinDirectory(repositoryName)); + } }; private static final String LINK_GUID = "58ec78bd-1176-4e36-8143-439f656b181d"; @@ -292,9 +343,11 @@ public ActionResult execute(ActionExecutionContext actionExecutionContext) private Spawn createSpawn(ActionExecutionContext actionExecutionContext) throws ActionExecutionException { try { + ArtifactExpander actionContextExpander = actionExecutionContext.getArtifactExpander(); + ArtifactExpander expander = actionContextExpander; return new SimpleSpawn( this, - ImmutableList.copyOf(getCommandLine(actionExecutionContext.getArtifactExpander())), + ImmutableList.copyOf(getCommandLine(expander)), getEffectiveEnvironment(actionExecutionContext.getClientEnv()), getExecutionInfo(), getInputs(), diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionBuilder.java index f364008afda6fe..b5c3816840042e 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionBuilder.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionBuilder.java @@ -68,27 +68,6 @@ /** Builder class to construct {@link CppLinkAction}s. */ public class CppLinkActionBuilder { - /** - * An implementation of {@link LinkArtifactFactory} that can create artifacts anywhere. - * - *

      Necessary when the LTO backend actions of libraries should be shareable, and thus cannot be - * under the package directory. - */ - static final LinkArtifactFactory SHAREABLE_LINK_ARTIFACT_FACTORY = - new LinkArtifactFactory() { - @Override - public Artifact create( - ActionConstructionContext actionConstructionContext, - RepositoryName repositoryName, - BuildConfigurationValue configuration, - PathFragment rootRelativePath) { - return actionConstructionContext.getShareableArtifact( - rootRelativePath, configuration.getBinDirectory(repositoryName)); - } - }; - - public static final String SHARED_NONLTO_BACKEND_ROOT_PREFIX = "shared.nonlto"; - private final Artifact output; private final CppSemantics cppSemantics; @Nullable private String mnemonic; @@ -147,6 +126,10 @@ public Artifact create( private boolean isStampingEnabled; private final Map executionInfo = new LinkedHashMap<>(); + // We have to add the dynamicLibrarySolibOutput to the CppLinkActionBuilder so that it knows how + // to set up the RPATH properly with respect to the symlink itself and not the original library. + private Artifact dynamicLibrarySolibSymlinkOutput; + /** * Creates a builder that builds {@link CppLinkAction}s. * @@ -330,6 +313,7 @@ private LtoBackendArtifacts createLtoArtifact( Artifact bitcodeFile, @Nullable BitcodeFiles allBitcode, PathFragment ltoOutputRootPrefix, + PathFragment ltoObjRootPrefix, boolean createSharedNonLto, List argv) throws RuleErrorException, InterruptedException { @@ -337,45 +321,28 @@ private LtoBackendArtifacts createLtoArtifact( // that will be fed the results of the indexing step, or a dummy LTO backend // that simply compiles the bitcode into native code without any index-based // cross module optimization. - Preconditions.checkArgument(actionConstructionContext instanceof RuleContext); - LtoBackendArtifacts ltoArtifact = - createSharedNonLto - ? new LtoBackendArtifacts( - ((RuleContext) actionConstructionContext).getStarlarkThread(), - ruleErrorConsumer, - configuration.getOptions(), - cppConfiguration, - ltoOutputRootPrefix, - bitcodeFile, - actionConstructionContext, - repositoryName, - configuration, - SHAREABLE_LINK_ARTIFACT_FACTORY, - featureConfiguration, - toolchain, - fdoContext, - usePicForLtoBackendActions, - toolchain.shouldCreatePerObjectDebugInfo(featureConfiguration, cppConfiguration), - argv) - : new LtoBackendArtifacts( - ((RuleContext) actionConstructionContext).getStarlarkThread(), - ruleErrorConsumer, - configuration.getOptions(), - cppConfiguration, - ltoOutputRootPrefix, - bitcodeFile, - allBitcode, - actionConstructionContext, - repositoryName, - configuration, - linkArtifactFactory, - featureConfiguration, - toolchain, - fdoContext, - usePicForLtoBackendActions, - toolchain.shouldCreatePerObjectDebugInfo(featureConfiguration, cppConfiguration), - argv); - return ltoArtifact; + LinkArtifactFactory linkFactory = + createSharedNonLto ? CppLinkAction.SHAREABLE_LINK_ARTIFACT_FACTORY : linkArtifactFactory; + BitcodeFiles bitcodeFiles = createSharedNonLto ? null : allBitcode; + return new LtoBackendArtifacts( + ((RuleContext) actionConstructionContext).getStarlarkThread(), + ruleErrorConsumer, + configuration.getOptions(), + cppConfiguration, + ltoOutputRootPrefix, + ltoObjRootPrefix, + bitcodeFile, + bitcodeFiles, + actionConstructionContext, + repositoryName, + configuration, + linkFactory, + featureConfiguration, + toolchain, + fdoContext, + usePicForLtoBackendActions, + toolchain.shouldCreatePerObjectDebugInfo(featureConfiguration, cppConfiguration), + argv); } private ImmutableList collectPerFileLtoBackendOpts(Artifact objectFile) { @@ -398,6 +365,7 @@ private List getLtoBackendUserCompileFlags( private Iterable createLtoArtifacts( PathFragment ltoOutputRootPrefix, + PathFragment ltoObjRootPrefix, NestedSet uniqueLibraries, boolean allowLtoIndexing, boolean includeLinkStaticInLtoIndexing) @@ -431,7 +399,11 @@ private Iterable createLtoArtifacts( } } BitcodeFiles bitcodeFiles = new BitcodeFiles(allBitcode.build()); - + if (bitcodeFiles.getFiles().toList().stream().anyMatch(Artifact::isTreeArtifact) + && ltoOutputRootPrefix.equals(ltoObjRootPrefix)) { + throw new RuleErrorException( + "Thinlto with tree artifacts requires feature use_lto_native_object_directory."); + } ImmutableList.Builder ltoOutputs = ImmutableList.builder(); for (LinkerInputs.LibraryToLink lib : uniqueLibraries.toList()) { if (!lib.containsObjectFiles()) { @@ -450,6 +422,7 @@ private Iterable createLtoArtifacts( objectFile, bitcodeFiles, ltoOutputRootPrefix, + ltoObjRootPrefix, /* createSharedNonLto= */ false, backendUserCompileFlags); ltoOutputs.add(ltoArtifacts); @@ -474,6 +447,7 @@ private Iterable createLtoArtifacts( input.getArtifact(), bitcodeFiles, ltoOutputRootPrefix, + ltoObjRootPrefix, !allowLtoIndexing, backendUserCompileFlags); ltoOutputs.add(ltoArtifacts); @@ -492,7 +466,11 @@ private ImmutableMap createSharedNonLtoArtifacts( return ImmutableMap.of(); } - PathFragment ltoOutputRootPrefix = PathFragment.create(SHARED_NONLTO_BACKEND_ROOT_PREFIX); + PathFragment ltoOutputRootPrefix = CppHelper.SHARED_NONLTO_BACKEND_ROOT_PREFIX; + PathFragment ltoObjRootPrefix = + featureConfiguration.isEnabled(CppRuleClasses.USE_LTO_NATIVE_OBJECT_DIRECTORY) + ? CppHelper.getThinLtoNativeObjectDirectoryFromLtoOutputRoot(ltoOutputRootPrefix) + : ltoOutputRootPrefix; ImmutableMap.Builder sharedNonLtoBackends = ImmutableMap.builder(); @@ -507,6 +485,7 @@ private ImmutableMap createSharedNonLtoArtifacts( input.getArtifact(), /* allBitcode= */ null, ltoOutputRootPrefix, + ltoObjRootPrefix, /* createSharedNonLto= */ true, backendUserCompileFlags); sharedNonLtoBackends.put(input.getArtifact(), ltoArtifacts); @@ -611,12 +590,17 @@ public CppLinkAction build() throws InterruptedException, RuleErrorException { || (linkingMode == Link.LinkingMode.DYNAMIC && !ltoCompilationContext.isEmpty()); PathFragment ltoOutputRootPrefix = null; + PathFragment ltoObjRootPrefix = null; if (isLtoIndexing) { Preconditions.checkState(allLtoArtifacts == null); ltoOutputRootPrefix = allowLtoIndexing - ? FileSystemUtils.appendExtension(output.getRootRelativePath(), ".lto") - : PathFragment.create(SHARED_NONLTO_BACKEND_ROOT_PREFIX); + ? CppHelper.getLtoOutputRootPrefix(output.getRootRelativePath()) + : CppHelper.SHARED_NONLTO_BACKEND_ROOT_PREFIX; + ltoObjRootPrefix = + featureConfiguration.isEnabled(CppRuleClasses.USE_LTO_NATIVE_OBJECT_DIRECTORY) + ? CppHelper.getThinLtoNativeObjectDirectoryFromLtoOutputRoot(ltoOutputRootPrefix) + : ltoOutputRootPrefix; // Use the originalUniqueLibraries which contains the full bitcode files // needed by the LTO backends (as opposed to the minimized bitcode files // containing just the summaries and symbol information that can be used by @@ -624,6 +608,7 @@ public CppLinkAction build() throws InterruptedException, RuleErrorException { allLtoArtifacts = createLtoArtifacts( ltoOutputRootPrefix, + ltoObjRootPrefix, originalUniqueLibraries, allowLtoIndexing, includeLinkStaticInLtoIndexing); @@ -814,7 +799,8 @@ public CppLinkAction build() throws InterruptedException, RuleErrorException { nonExpandedLinkerInputs, needWholeArchive, ruleErrorConsumer, - ((RuleContext) actionConstructionContext).getWorkspaceName()); + ((RuleContext) actionConstructionContext).getWorkspaceName(), + dynamicLibrarySolibSymlinkOutput); CollectedLibrariesToLink collectedLibrariesToLink = librariesToLinkCollector.collectLibrariesToLink(); @@ -831,13 +817,6 @@ public CppLinkAction build() throws InterruptedException, RuleErrorException { userLinkFlags.addAll(cppConfiguration.getLtoIndexOptions()); } - NestedSet runtimeLibrarySearchDirectories = - collectedLibrariesToLink.getRuntimeLibrarySearchDirectories(); - if (linkType.getActionName().equals(CppActionNames.CPP_LINK_DYNAMIC_LIBRARY) - && featureConfiguration.isEnabled( - CppRuleClasses.EXCLUDE_BAZEL_RPATHS_IN_TRANSITIVE_LIBS_FEATURE_NAME)) { - runtimeLibrarySearchDirectories = null; - } variables = LinkBuildVariables.setupVariables( ((RuleContext) actionConstructionContext).getStarlarkThread(), @@ -863,9 +842,10 @@ public CppLinkAction build() throws InterruptedException, RuleErrorException { toolchain.getInterfaceSoBuilder().getExecPathString(), interfaceOutput != null ? interfaceOutput.getExecPathString() : null, ltoOutputRootPrefix, + ltoObjRootPrefix, defFile != null ? defFile.getExecPathString() : null, fdoContext, - runtimeLibrarySearchDirectories, + collectedLibrariesToLink.getRuntimeLibrarySearchDirectories(), collectedLibrariesToLink.getLibrariesToLink(), collectedLibrariesToLink.getLibrarySearchDirectories(), /* addIfsoRelatedVariables= */ true); @@ -1563,4 +1543,11 @@ public CppLinkActionBuilder addExecutionInfo(Map executionInfo) this.executionInfo.putAll(executionInfo); return this; } + + @CanIgnoreReturnValue + public CppLinkActionBuilder setDynamicLibrarySolibSymlinkOutput( + Artifact dynamicLibrarySolibSymlinkOutput) { + this.dynamicLibrarySolibSymlinkOutput = dynamicLibrarySolibSymlinkOutput; + return this; + } } diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java index fab3d348f40815..178048f3b553f1 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java @@ -287,6 +287,9 @@ public static ToolchainTypeRequirement ccToolchainTypeRequirement(RuleDefinition /** A string constant for the LTO indexing bitcode feature. */ public static final String NO_USE_LTO_INDEXING_BITCODE_FILE = "no_use_lto_indexing_bitcode_file"; + /** A string constant for the LTO separate native object directory feature. */ + public static final String USE_LTO_NATIVE_OBJECT_DIRECTORY = "use_lto_native_object_directory"; + /* * A string constant for allowing implicit ThinLTO enablement for AFDO. */ @@ -517,37 +520,6 @@ public static ToolchainTypeRequirement ccToolchainTypeRequirement(RuleDefinition */ public static final String LEGACY_IS_CC_TEST_FEATURE_NAME = "legacy_is_cc_test"; - /** - * By default Bazel will be embed runtime search directories (RPATHS) in transitive shared - * libraries, however, for Linux they are wrong in most cases since the runfiles directory where - * the transitive shared library will live will not be known at the time of linking. The runfiles - * directory is decided by dependent rules, we don't know where those dependents will live. For - * Mac (where it works differently by searching from the library's path instead of the main - * binary's) the loader paths (not rpaths) are correct so the paths will work. - * - *

      This feature controls whether Bazel will embed those rpaths into the transitive shared - * library. - */ - public static final String EXCLUDE_BAZEL_RPATHS_IN_TRANSITIVE_LIBS_FEATURE_NAME = - "exclude_bazel_rpaths_in_transitive_libs"; - - /** - * With this feature enabled cc_binary will link all its dynamic_deps, even the ones it depends on - * transitively, linking indirect deps might be necessary because if the RPATHs haven't been set - * up properly in those dynamic_deps then the loader won't be able to find those libraries unless - * they are also linked. For a production binary this is probably not the desired behavior and you - * can switch it off by disabling this feature, for the binary to work you have to make sure that - * the RPATHs in all shared libraries are set up properly though. The default toolchains have this - * behavior switched off for cc_binaries by default. The behavior for cc_tests with dynamic_deps - * on all platforms and Windows cc_binaries is hardcoded to always link every transitive library. - * - *

      This feature controls the behavior for shared libraries depended on via dynamic_deps and - * doesn't control the behavior of the dynamic dependencies created by cc_libraries and used for - * cc_tests or cc_binaries(linkstatic=0). - */ - public static final String LINK_INDIRECT_DYNAMIC_DEPS_IN_BINARY_FEATURE_NAME = - "link_indirect_dynamic_deps_in_binary"; - /** Ancestor for all rules that do include scanning. */ public static final class CcIncludeScanningRule implements RuleDefinition { private final boolean addGrepIncludes; diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/GraphNodeAspect.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/GraphNodeAspect.java deleted file mode 100644 index 68d42d9929f1dc..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/GraphNodeAspect.java +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2019 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.lib.rules.cpp; - - -import com.google.common.base.Function; -import com.google.common.collect.ImmutableList; -import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException; -import com.google.devtools.build.lib.analysis.AnalysisUtils; -import com.google.devtools.build.lib.analysis.ConfiguredAspect; -import com.google.devtools.build.lib.analysis.ConfiguredAspectFactory; -import com.google.devtools.build.lib.analysis.ConfiguredTarget; -import com.google.devtools.build.lib.analysis.RuleContext; -import com.google.devtools.build.lib.cmdline.Label; -import com.google.devtools.build.lib.cmdline.RepositoryName; -import com.google.devtools.build.lib.packages.AspectDefinition; -import com.google.devtools.build.lib.packages.AspectParameters; -import com.google.devtools.build.lib.packages.NativeAspectClass; -import com.google.devtools.build.lib.packages.Rule; -import javax.annotation.Nullable; - -/** - * Aspect for constructing a tree of labels that is used to prune static libraries that are already - * linked dynamically into a cc_binary. TODO(b/145508948): Try to remove this class in the future. - */ -public final class GraphNodeAspect extends NativeAspectClass implements ConfiguredAspectFactory { - // When the dynamic_deps attribute is not set, we return null. We would only want the graph to be - // analyzed with the aspect in the cases that we have set dynamic_deps. Otherwise it would be a - // waste of memory in the cases where we don't need the aspect. If we return null, the aspect is - // not used analyze anything. - // See - // https://github.com/bazelbuild/bazel/blob/df52777aac8cbfc7719af9f0dbb23335e59c42df/src/main/java/com/google/devtools/build/lib/packages/Attribute.java#L114 - public static final Function ASPECT_PARAMETERS = - new Function() { - @Nullable - @Override - public AspectParameters apply(Rule rule) { - return rule.isAttributeValueExplicitlySpecified("dynamic_deps") - ? AspectParameters.EMPTY - : null; - } - }; - - @Override - public AspectDefinition getDefinition(AspectParameters aspectParameters) { - return new AspectDefinition.Builder(this) - .propagateAlongAllAttributes() - .requireStarlarkProviders(CcInfo.PROVIDER.id()) - .build(); - } - - @Override - public ConfiguredAspect create( - Label targetLabel, - ConfiguredTarget ct, - RuleContext ruleContext, - AspectParameters params, - RepositoryName toolsRepository) - throws ActionConflictException, InterruptedException { - ImmutableList.Builder children = ImmutableList.builder(); - if (ruleContext.attributes().has("deps")) { - children.addAll( - AnalysisUtils.getProviders(ruleContext.getPrerequisites("deps"), GraphNodeInfo.class)); - } - return new ConfiguredAspect.Builder(ruleContext) - .addProvider( - GraphNodeInfo.class, new GraphNodeInfo(ruleContext.getLabel(), children.build())) - .build(); - } -} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/GraphNodeInfo.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/GraphNodeInfo.java deleted file mode 100644 index 9c663eddb79cf4..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/GraphNodeInfo.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2019 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.lib.rules.cpp; - -import com.google.common.collect.ImmutableList; -import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; -import com.google.devtools.build.lib.cmdline.Label; -import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; -import java.util.List; - -/** Provider used to propagate information for {@link GraphNodeAspect}. */ -@Immutable -public final class GraphNodeInfo implements TransitiveInfoProvider { - private final Label label; - private final ImmutableList children; - - public GraphNodeInfo(Label label, List children) { - this.label = label; - this.children = children == null ? null : ImmutableList.copyOf(children); - } - - public Label getLabel() { - return label; - } - - public List getChildren() { - return children; - } -} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LibrariesToLinkCollector.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LibrariesToLinkCollector.java index abf3c1babf78be..5fce318f621219 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/LibrariesToLinkCollector.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LibrariesToLinkCollector.java @@ -61,6 +61,7 @@ public class LibrariesToLinkCollector { private final RuleErrorConsumer ruleErrorConsumer; private final Artifact output; private final String workspaceName; + private final Artifact dynamicLibrarySolibSymlinkOutput; public LibrariesToLinkCollector( boolean isNativeDeps, @@ -79,7 +80,8 @@ public LibrariesToLinkCollector( Iterable linkerInputs, boolean needWholeArchive, RuleErrorConsumer ruleErrorConsumer, - String workspaceName) { + String workspaceName, + Artifact dynamicLibrarySolibSymlinkOutput) { this.isNativeDeps = isNativeDeps; this.cppConfiguration = cppConfiguration; this.ccToolchainProvider = toolchain; @@ -95,6 +97,7 @@ public LibrariesToLinkCollector( this.ruleErrorConsumer = ruleErrorConsumer; this.output = output; this.workspaceName = workspaceName; + this.dynamicLibrarySolibSymlinkOutput = dynamicLibrarySolibSymlinkOutput; needToolchainLibrariesRpath = toolchainLibrariesSolibDir != null @@ -164,81 +167,88 @@ private NestedSet collectToolchainRuntimeLibrarySearchDirectories( } private ImmutableList findPotentialSolibParents() { - // The runtime location of the solib directory relative to the binary depends on four factors: - // - // * whether the binary is contained in the main repository or an external repository; - // * whether the binary is executed directly or from a runfiles tree; - // * whether the binary is staged as a symlink (sandboxed execution; local execution if the - // binary is in the runfiles of another target) or a regular file (remote execution) - the - // dynamic linker follows sandbox and runfiles symlinks into its location under the - // unsandboxed execroot, which thus becomes the effective $ORIGIN; - // * whether --experimental_sibling_repository_layout is enabled or not. - // - // The rpaths emitted into the binary thus have to cover the following cases (assuming that - // the binary target is located in the pkg `pkg` and has name `file`) for the directory used - // as $ORIGIN by the dynamic linker and the directory containing the solib directories: - // - // 1. main, direct, symlink: - // $ORIGIN: $EXECROOT/pkg - // solib root: $EXECROOT - // 2. main, direct, regular file: - // $ORIGIN: $EXECROOT/pkg - // solib root: $EXECROOT/pkg/file.runfiles/main_repo - // 3. main, runfiles, symlink: - // $ORIGIN: $EXECROOT/pkg - // solib root: $EXECROOT - // 4. main, runfiles, regular file: - // $ORIGIN: other_target.runfiles/main_repo/pkg - // solib root: other_target.runfiles/main_repo - // 5a. external, direct, symlink: - // $ORIGIN: $EXECROOT/external/other_repo/pkg - // solib root: $EXECROOT - // 5b. external, direct, symlink, with --experimental_sibling_repository_layout: - // $ORIGIN: $EXECROOT/../other_repo/pkg - // solib root: $EXECROOT/../other_repo - // 6a. external, direct, regular file: - // $ORIGIN: $EXECROOT/external/other_repo/pkg - // solib root: $EXECROOT/external/other_repo/pkg/file.runfiles/main_repo - // 6b. external, direct, regular file, with --experimental_sibling_repository_layout: - // $ORIGIN: $EXECROOT/../other_repo/pkg - // solib root: $EXECROOT/../other_repo/pkg/file.runfiles/other_repo - // 7a. external, runfiles, symlink: - // $ORIGIN: $EXECROOT/external/other_repo/pkg - // solib root: $EXECROOT - // 7b. external, runfiles, symlink, with --experimental_sibling_repository_layout: - // $ORIGIN: $EXECROOT/../other_repo/pkg - // solib root: $EXECROOT/../other_repo - // 8a. external, runfiles, regular file: - // $ORIGIN: other_target.runfiles/some_repo/pkg - // solib root: other_target.runfiles/main_repo - // 8b. external, runfiles, regular file, with --experimental_sibling_repository_layout: - // $ORIGIN: other_target.runfiles/some_repo/pkg - // solib root: other_target.runfiles/some_repo - // - // Cases 1, 3, 4, 5, 7, and 8b are covered by an rpath that walks up the root relative path. - // Cases 2 and 6 covered by walking into file.runfiles/main_repo. - // Case 8a is covered by walking up some_repo/pkg and then into main_repo. - boolean isExternal = - output.getRunfilesPath().startsWith(LabelConstants.EXTERNAL_RUNFILES_PATH_PREFIX); - boolean usesLegacyRepositoryLayout = output.getRoot().isLegacy(); ImmutableList.Builder solibParents = ImmutableList.builder(); - // Handles cases 1, 3, 4, 5, and 7. - solibParents.add("../".repeat(output.getRootRelativePath().segmentCount() - 1)); - // Handle cases 2 and 6. - String solibRepositoryName; - if (isExternal && !usesLegacyRepositoryLayout) { - // Case 6b - solibRepositoryName = output.getRunfilesPath().getSegment(1); - } else { - // Cases 2 and 6a - solibRepositoryName = workspaceName; + ImmutableList.Builder outputs = ImmutableList.builder(); + outputs.add(output); + if (dynamicLibrarySolibSymlinkOutput != null) { + outputs.add(dynamicLibrarySolibSymlinkOutput); } - solibParents.add(output.getFilename() + ".runfiles/" + solibRepositoryName + "/"); - if (isExternal && usesLegacyRepositoryLayout) { - // Handles case 8a. The runfiles path is of the form ../some_repo/pkg/file and we need to - // walk up some_repo/pkg and then down into main_repo. - solibParents.add( - "../".repeat(output.getRunfilesPath().segmentCount() - 2) + workspaceName + "/"); + for (Artifact output : outputs.build()) { + // The runtime location of the solib directory relative to the binary depends on four factors: + // + // * whether the binary is contained in the main repository or an external repository; + // * whether the binary is executed directly or from a runfiles tree; + // * whether the binary is staged as a symlink (sandboxed execution; local execution if the + // binary is in the runfiles of another target) or a regular file (remote execution) - the + // dynamic linker follows sandbox and runfiles symlinks into its location under the + // unsandboxed execroot, which thus becomes the effective $ORIGIN; + // * whether --experimental_sibling_repository_layout is enabled or not. + // + // The rpaths emitted into the binary thus have to cover the following cases (assuming that + // the binary target is located in the pkg `pkg` and has name `file`) for the directory used + // as $ORIGIN by the dynamic linker and the directory containing the solib directories: + // + // 1. main, direct, symlink: + // $ORIGIN: $EXECROOT/pkg + // solib root: $EXECROOT + // 2. main, direct, regular file: + // $ORIGIN: $EXECROOT/pkg + // solib root: $EXECROOT/pkg/file.runfiles/main_repo + // 3. main, runfiles, symlink: + // $ORIGIN: $EXECROOT/pkg + // solib root: $EXECROOT + // 4. main, runfiles, regular file: + // $ORIGIN: other_target.runfiles/main_repo/pkg + // solib root: other_target.runfiles/main_repo + // 5a. external, direct, symlink: + // $ORIGIN: $EXECROOT/external/other_repo/pkg + // solib root: $EXECROOT + // 5b. external, direct, symlink, with --experimental_sibling_repository_layout: + // $ORIGIN: $EXECROOT/../other_repo/pkg + // solib root: $EXECROOT/../other_repo + // 6a. external, direct, regular file: + // $ORIGIN: $EXECROOT/external/other_repo/pkg + // solib root: $EXECROOT/external/other_repo/pkg/file.runfiles/main_repo + // 6b. external, direct, regular file, with --experimental_sibling_repository_layout: + // $ORIGIN: $EXECROOT/../other_repo/pkg + // solib root: $EXECROOT/../other_repo/pkg/file.runfiles/other_repo + // 7a. external, runfiles, symlink: + // $ORIGIN: $EXECROOT/external/other_repo/pkg + // solib root: $EXECROOT + // 7b. external, runfiles, symlink, with --experimental_sibling_repository_layout: + // $ORIGIN: $EXECROOT/../other_repo/pkg + // solib root: $EXECROOT/../other_repo + // 8a. external, runfiles, regular file: + // $ORIGIN: other_target.runfiles/some_repo/pkg + // solib root: other_target.runfiles/main_repo + // 8b. external, runfiles, regular file, with --experimental_sibling_repository_layout: + // $ORIGIN: other_target.runfiles/some_repo/pkg + // solib root: other_target.runfiles/some_repo + // + // Cases 1, 3, 4, 5, 7, and 8b are covered by an rpath that walks up the root relative path. + // Cases 2 and 6 covered by walking into file.runfiles/main_repo. + // Case 8a is covered by walking up some_repo/pkg and then into main_repo. + boolean isExternal = + output.getRunfilesPath().startsWith(LabelConstants.EXTERNAL_RUNFILES_PATH_PREFIX); + boolean usesLegacyRepositoryLayout = output.getRoot().isLegacy(); + // Handles cases 1, 3, 4, 5, and 7. + solibParents.add("../".repeat(output.getRootRelativePath().segmentCount() - 1)); + // Handle cases 2 and 6. + String solibRepositoryName; + if (isExternal && !usesLegacyRepositoryLayout) { + // Case 6b + solibRepositoryName = output.getRunfilesPath().getSegment(1); + } else { + // Cases 2 and 6a + solibRepositoryName = workspaceName; + } + solibParents.add(output.getFilename() + ".runfiles/" + solibRepositoryName + "/"); + if (isExternal && usesLegacyRepositoryLayout) { + // Handles case 8a. The runfiles path is of the form ../some_repo/pkg/file and we need to + // walk up some_repo/pkg and then down into main_repo. + solibParents.add( + "../".repeat(output.getRunfilesPath().segmentCount() - 2) + workspaceName + "/"); + } } return solibParents.build(); @@ -768,9 +778,7 @@ private static boolean handledByLtoIndexing(Artifact a, boolean allowLtoIndexing // Otherwise, this may be from a linkstatic library that we decided not to include in // LTO indexing because we are linking a test, to improve scalability when linking many tests. return allowLtoIndexing - && !a.getRootRelativePath() - .startsWith( - PathFragment.create(CppLinkActionBuilder.SHARED_NONLTO_BACKEND_ROOT_PREFIX)); + && !a.getRootRelativePath().startsWith(CppHelper.SHARED_NONLTO_BACKEND_ROOT_PREFIX); } @Nullable diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkBuildVariables.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkBuildVariables.java index 8e5abc61ede9b5..2f2c1a17ebe3ac 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkBuildVariables.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkBuildVariables.java @@ -118,6 +118,7 @@ public static CcToolchainVariables setupVariables( String interfaceLibraryBuilder, String interfaceLibraryOutput, PathFragment ltoOutputRootPrefix, + PathFragment ltoObjRootPrefix, String defFile, FdoContext fdoContext, NestedSet runtimeLibrarySearchDirectories, @@ -185,13 +186,27 @@ public static CcToolchainVariables setupVariables( // TODO(b/33846234): Remove once all the relevant crosstools don't depend on the variable. buildVariables.addStringVariable("thinlto_optional_params_file", ""); } - // Given "fullbitcode_prefix;thinlto_index_prefix", replaces fullbitcode_prefix with - // thinlto_index_prefix to generate the index and imports files. + // Given "fullbitcode_prefix;thinlto_index_prefix;native_object_prefix", replaces + // fullbitcode_prefix with thinlto_index_prefix to generate the index and imports files. // fullbitcode_prefix is the empty string because we are appending a prefix to the fullbitcode // instead of replacing it. This argument is passed to the linker. - buildVariables.addStringVariable( - THINLTO_PREFIX_REPLACE.getVariableName(), - ";" + binDirectoryPath.getRelative(ltoOutputRootPrefix) + '/'); + // The native objects generated after the LTOBackend action are stored in a directory by + // replacing the prefix "fullbitcode_prefix" with "native_object_prefix", and this is used + // when generating the param file in the indexing step, which will be used during the final + // link step. + if (!ltoOutputRootPrefix.equals(ltoObjRootPrefix)) { + buildVariables.addStringVariable( + THINLTO_PREFIX_REPLACE.getVariableName(), + ";" + + binDirectoryPath.getRelative(ltoOutputRootPrefix) + + "/;" + + binDirectoryPath.getRelative(ltoObjRootPrefix) + + "/"); + } else { + buildVariables.addStringVariable( + THINLTO_PREFIX_REPLACE.getVariableName(), + ";" + binDirectoryPath.getRelative(ltoOutputRootPrefix) + "/"); + } String objectFileExtension = ccToolchainProvider .getFeatures() diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendAction.java index 86d6decae305af..840924be8cee36 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendAction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendAction.java @@ -14,14 +14,12 @@ package com.google.devtools.build.lib.rules.cpp; -import static com.google.common.collect.ImmutableSet.toImmutableSet; import static java.util.stream.Collectors.joining; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; import com.google.devtools.build.lib.actions.AbstractAction; import com.google.devtools.build.lib.actions.ActionEnvironment; import com.google.devtools.build.lib.actions.ActionExecutionContext; @@ -44,11 +42,13 @@ import com.google.devtools.build.lib.util.DetailedExitCode; import com.google.devtools.build.lib.util.Fingerprint; import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.IOException; import java.util.HashSet; import java.util.Map; +import java.util.Optional; import java.util.Set; import javax.annotation.Nullable; @@ -122,13 +122,60 @@ protected void setInputsDiscovered(boolean inputsDiscovered) { this.inputsDiscovered = inputsDiscovered; } - private NestedSet computeBitcodeInputs(HashSet inputPaths) { + /** + * Given a map of path to artifact, and a path, returns the artifact whose key is in the map, or + * if none, an artifact whose key matches a prefix of the path. Assumes that artifacts whose paths + * are directories are tree artifacts. Assumes that no artifact key is a sub directory of another + * artifact key. For example, "path/file1" may return the artifact whose path is "path/file1" or + * whose path is "path/". Returns empty if there are no matches. + */ + private Optional getArtifactOrTreeArtifact( + PathFragment path, Map pathToArtifact) { + PathFragment currentPath = path; + while (!currentPath.isEmpty()) { + if (pathToArtifact.containsKey(currentPath)) { + return Optional.of(pathToArtifact.get(currentPath)); + } else { + currentPath = currentPath.getParentDirectory(); + } + } + return Optional.empty(); + } + + /** + * Throws an error if any of the input paths is not in the bitcodeFiles or in a subdirecorty of a + * file in bitcodeFiles + */ + private NestedSet computeBitcodeInputs( + HashSet inputPaths, ActionExecutionContext actionExecutionContext) + throws ActionExecutionException { NestedSetBuilder bitcodeInputs = NestedSetBuilder.stableOrder(); - for (Artifact inputArtifact : bitcodeFiles.getFiles().toList()) { - if (inputPaths.contains(inputArtifact.getExecPath())) { - bitcodeInputs.add(inputArtifact); + ImmutableMap execPathToArtifact = + bitcodeFiles.getFilesArtifactPathMap(); + Set missingInputs = new HashSet<>(); + for (PathFragment inputPath : inputPaths) { + Optional maybeArtifact = getArtifactOrTreeArtifact(inputPath, execPathToArtifact); + if (maybeArtifact.isPresent()) { + bitcodeInputs.add(maybeArtifact.get()); + } else { + // One of the inputs is not present. We add it to missingInputs and will fail. + missingInputs.add(inputPath); } } + if (!missingInputs.isEmpty()) { + String message = + String.format( + "error computing inputs from imports file: %s, missing bitcode files (first 10): %s", + actionExecutionContext.getInputPath(imports), + // Limit the reported count to protect against a large error message. + missingInputs.stream() + .map(Object::toString) + .sorted() + .limit(10) + .collect(joining(", "))); + DetailedExitCode code = createDetailedExitCode(message, Code.MISSING_BITCODE_FILES); + throw new ActionExecutionException(message, this, false, code); + } return bitcodeInputs.build(); } @@ -136,9 +183,10 @@ private NestedSet computeBitcodeInputs(HashSet inputPath @Override public NestedSet discoverInputs(ActionExecutionContext actionExecutionContext) throws ActionExecutionException { + Path importsFilePath = actionExecutionContext.getInputPath(imports); ImmutableList lines; try { - lines = FileSystemUtils.readLinesAsLatin1(actionExecutionContext.getInputPath(imports)); + lines = FileSystemUtils.readLinesAsLatin1(importsFilePath); } catch (IOException e) { String message = String.format( @@ -168,27 +216,8 @@ public NestedSet discoverInputs(ActionExecutionContext actionExecution } // Convert the import set of paths to the set of bitcode file artifacts. - NestedSet bitcodeInputSet = computeBitcodeInputs(importSet); - if (bitcodeInputSet.memoizedFlattenAndGetSize() != importSet.size()) { - Set missingInputs = - Sets.difference( - importSet, - bitcodeInputSet.toList().stream() - .map(Artifact::getExecPath) - .collect(toImmutableSet())); - String message = - String.format( - "error computing inputs from imports file: %s, missing bitcode files (first 10): %s", - actionExecutionContext.getInputPath(imports), - // Limit the reported count to protect against a large error message. - missingInputs.stream() - .map(Object::toString) - .sorted() - .limit(10) - .collect(joining(", "))); - DetailedExitCode code = createDetailedExitCode(message, Code.MISSING_BITCODE_FILES); - throw new ActionExecutionException(message, this, false, code); - } + // Throws an error if there is any path in the importset that is not pat of any artifact + NestedSet bitcodeInputSet = computeBitcodeInputs(importSet, actionExecutionContext); updateInputs( NestedSetBuilder.fromNestedSet(bitcodeInputSet).addTransitive(mandatoryInputs).build()); return bitcodeInputSet; @@ -251,6 +280,16 @@ public static class Builder extends SpawnAction.Builder { private BitcodeFiles bitcodeFiles; private Artifact imports; + public Builder() { + super(); + } + + public Builder(Builder other) { + super(other); + bitcodeFiles = other.bitcodeFiles; + imports = other.imports; + } + @CanIgnoreReturnValue public Builder addImportsInfo(BitcodeFiles allBitcodeFiles, Artifact importsFile) { this.bitcodeFiles = allBitcodeFiles; diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendActionTemplate.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendActionTemplate.java new file mode 100644 index 00000000000000..0160101bc3595f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendActionTemplate.java @@ -0,0 +1,387 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.actions.ActionAnalysisMetadata; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionKeyCacher; +import com.google.devtools.build.lib.actions.ActionKeyContext; +import com.google.devtools.build.lib.actions.ActionLookupKey; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.ActionTemplate; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact; +import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact; +import com.google.devtools.build.lib.actions.CommandLineExpansionException; +import com.google.devtools.build.lib.actions.MiddlemanType; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; +import com.google.devtools.build.lib.server.FailureDetails; +import com.google.devtools.build.lib.util.DetailedExitCode; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; +import javax.annotation.Nullable; + +/** + * An {@link ActionTemplate} that expands into {@link LtoBackendAction}s at execution time. Is is + * similar to {@link com.google.devtools.build.lib.analysis.actions.SpawnActionTemplate}. + */ +public final class LtoBackendActionTemplate extends ActionKeyCacher + implements ActionTemplate { + private final LtoBackendAction.Builder ltoBackendActionbuilder; + private final CcToolchainVariables buildVariables; + + // An input tree artifact containing the full bitcode. It is never null. + private final SpecialArtifact fullBitcodeTreeArtifact; + + // An input tree artifact containing ".thinlto.bc" and ".imports" files, generated together with + // It will be null when this is a shared non-lto backend. + @Nullable private final SpecialArtifact indexAndImportsTreeArtifact; + + // An output tree artifact that will contain the native objects. In a sibling directory to + // indexTreeArtifact. The objects will be generated in the same location as defined in the .param + // file created during the lto indexing step. + private final SpecialArtifact objectFileTreeArtifact; + + // The corresponding dwoFile if fission is used. + private final SpecialArtifact dwoFileTreeArtifact; + + private final FeatureConfiguration featureConfiguration; + + private final boolean usePic; + + private final BitcodeFiles bitcodeFiles; + + private final ActionOwner actionOwner; + private final NestedSet mandatoryInputs; + private final NestedSet allInputs; + + /** + * Creates an LtoBackendActionTemplate. + * + * @param indexAndImportsTreeArtifact the TreeArtifact that contains .thinlto.bc. and .imports + * files. + * @param fullBitcodeTreeArtifact the TreeArtifact that contains .pic.o files. + * @param objectFileTreeArtifact the TreeArtifact that contains .pic.o files. + * @param dwoFileTreeArtifact the TreeArtifact that contains .dwo files. + * @param featureConfiguration the feature configuration. + * @param ltoBackendActionbuilder An almost completely configured {@link LtoBackendAction.Builder} + * without the input and output files set. It is used as a template to instantiate expanded + * {@link LtoBackendAction}s. + * @param buildVariables the building variables. + * @param usePic whether to use PIC or not. + * @param actionOwner the owner of this {@link ActionTemplate}. + */ + LtoBackendActionTemplate( + SpecialArtifact indexAndImportsTreeArtifact, + SpecialArtifact fullBitcodeTreeArtifact, + SpecialArtifact objectFileTreeArtifact, + SpecialArtifact dwoFileTreeArtifact, + FeatureConfiguration featureConfiguration, + LtoBackendAction.Builder ltoBackendActionbuilder, + CcToolchainVariables buildVariables, + boolean usePic, + BitcodeFiles bitcodeFiles, + ActionOwner actionOwner) { + this.ltoBackendActionbuilder = ltoBackendActionbuilder; + this.buildVariables = buildVariables; + this.indexAndImportsTreeArtifact = indexAndImportsTreeArtifact; + this.fullBitcodeTreeArtifact = fullBitcodeTreeArtifact; + this.objectFileTreeArtifact = objectFileTreeArtifact; + this.dwoFileTreeArtifact = dwoFileTreeArtifact; + this.actionOwner = checkNotNull(actionOwner, objectFileTreeArtifact); + this.featureConfiguration = featureConfiguration; + this.usePic = usePic; + this.bitcodeFiles = bitcodeFiles; + + NestedSetBuilder mandatoryInputsBuilder = + NestedSetBuilder.compileOrder() + .add(fullBitcodeTreeArtifact) + .addTransitive(ltoBackendActionbuilder.getInputsAndTools()); + if (indexAndImportsTreeArtifact != null) { + mandatoryInputsBuilder.add(indexAndImportsTreeArtifact); + } + this.mandatoryInputs = mandatoryInputsBuilder.build(); + this.allInputs = mandatoryInputs; + } + + /** Helper functions for generateActionsForInputArtifacts */ + private String pathFragmentToRelativePath(PathFragment parentPath, PathFragment path) { + return path.relativeTo(parentPath).getSafePathString(); + } + + private String removeImportsExtension(String path) { + return FileSystemUtils.removeExtension(path); + } + + private String removeThinltoBcExtension(String path) { + return FileSystemUtils.removeExtension(FileSystemUtils.removeExtension(path)); + } + + /** + * Given all the files inside indexAndImportsTreeArtifact, we find the corresponding index and + * imports files. Then we use their path together with the fullBitcodeTreeArtifact path to derive + * the path of the original full bitcode file. Then for each imports file, we create an lto + * backend action that depends on that import file, on the corresponding index file, and on the + * whole fullBitcodeTreeArtifact, which it uses to find the full bitcode file. TODO(antunesi): + * make the generated action depend only on the corresponding full bitcode file rather than depend + * on the whole tree artifact that contains the full bitcode file. + */ + @Override + public ImmutableList generateActionsForInputArtifacts( + ImmutableSet inputTreeFileArtifacts, ActionLookupKey artifactOwner) + throws ActionExecutionException { + ImmutableList.Builder expandedActions = new ImmutableList.Builder<>(); + + final FileType thinltoBcSourceType = CppFileTypes.LTO_INDEXING_ANALYSIS_FILE; + final FileType importsType = CppFileTypes.LTO_IMPORTS_FILE; + + ImmutableList.Builder importsBuilder = ImmutableList.builder(); + ImmutableMap.Builder nameToThinLtoBuilder = + new ImmutableMap.Builder<>(); + + PathFragment indexAndImportParentPath = indexAndImportsTreeArtifact.getExecPath(); + + for (TreeFileArtifact inputTreeFileArtifact : inputTreeFileArtifacts) { + PathFragment path = inputTreeFileArtifact.getExecPath(); + boolean isThinLto = thinltoBcSourceType.matches(path); + boolean isImport = importsType.matches(path); + + if (isThinLto) { + String thinLtoNoExtension = + removeThinltoBcExtension(pathFragmentToRelativePath(indexAndImportParentPath, path)); + nameToThinLtoBuilder.put(thinLtoNoExtension, inputTreeFileArtifact); + } else if (isImport) { + importsBuilder.add(inputTreeFileArtifact); + } else { + String message = + String.format( + "Artifact '%s' expanded from the directory artifact '%s' is neither imports nor" + + " thinlto .", + inputTreeFileArtifact.getExecPathString(), + fullBitcodeTreeArtifact.getExecPathString()); // kinda wrong, should be index + throw new ActionExecutionException( + message, this, /* catastrophe= */ false, makeDetailedExitCode(message)); + } + } + + // Maps each imports to a .bc file + ImmutableList imports = importsBuilder.build(); + ImmutableMap nameToThinLto = nameToThinLtoBuilder.buildOrThrow(); + if (imports.size() != nameToThinLto.size()) { + String message = + String.format( + "Either both or neither bitcodeFiles and imports files should be null. %s %s" + ".", + inputTreeFileArtifacts, + fullBitcodeTreeArtifact.getExecPathString()); // kinda wrong, should be index + throw new ActionExecutionException( + message, this, /* catastrophe= */ false, makeDetailedExitCode(message)); + } + + for (TreeFileArtifact importFile : imports) { + PathFragment path = importFile.getExecPath(); + String relativePathNoExtension = + removeImportsExtension(pathFragmentToRelativePath(indexAndImportParentPath, path)); + TreeFileArtifact thinLtoFile = nameToThinLto.get(relativePathNoExtension); + PathFragment fullBitcodePath = + fullBitcodeTreeArtifact.getExecPath().getRelative(relativePathNoExtension); + String outputName = relativePathNoExtension; + TreeFileArtifact objTreeFileArtifact = + TreeFileArtifact.createTemplateExpansionOutput( + objectFileTreeArtifact, outputName, artifactOwner); + TreeFileArtifact dwoFileArtifact = null; + if (dwoFileTreeArtifact != null) { + dwoFileArtifact = + TreeFileArtifact.createTemplateExpansionOutput( + dwoFileTreeArtifact, + FileSystemUtils.replaceExtension( + PathFragment.create(relativePathNoExtension), ".dwo"), + artifactOwner); + } + LtoBackendAction.Builder builderCopy = new LtoBackendAction.Builder(ltoBackendActionbuilder); + + LtoBackendArtifacts.addArtifactsLtoBackendAction( + builderCopy, + buildVariables, + featureConfiguration, + thinLtoFile, + importFile, + fullBitcodeTreeArtifact, + objTreeFileArtifact, + bitcodeFiles, + dwoFileArtifact, + usePic, + fullBitcodePath.toString(), + /* isDummyAction= */ false); + expandedActions.add((LtoBackendAction) builderCopy.buildForActionTemplate(actionOwner)); + } + + return expandedActions.build(); + } + + @Override + protected void computeKey( + ActionKeyContext actionKeyContext, + @Nullable Artifact.ArtifactExpander artifactExpander, + Fingerprint fp) + throws CommandLineExpansionException, InterruptedException { + + LtoBackendAction dummyAction = getDummyAction(); + dummyAction.computeKey(actionKeyContext, artifactExpander, fp); + } + + /** + * This is an action that is not valid, because its input bitcode file is a TreeArtifact rather + * than a specific file. It is useful for calculating keys and inputs of the Action Template by + * reusing functionality from LtoBackendAction. + */ + private LtoBackendAction getDummyAction() { + LtoBackendAction.Builder builderCopy = new LtoBackendAction.Builder(ltoBackendActionbuilder); + // This is a dummy action that would not work, because the bitcode file path is a directory + // rather than a file. + LtoBackendArtifacts.addArtifactsLtoBackendAction( + builderCopy, + buildVariables, + featureConfiguration, + indexAndImportsTreeArtifact, + indexAndImportsTreeArtifact, + fullBitcodeTreeArtifact, + objectFileTreeArtifact, + bitcodeFiles, + dwoFileTreeArtifact, + usePic, + null, + /* isDummyAction= */ true); + + return (LtoBackendAction) builderCopy.buildForActionTemplate(actionOwner); + } + + @Override + public SpecialArtifact getInputTreeArtifact() { + return indexAndImportsTreeArtifact; + } + + @Override + public SpecialArtifact getOutputTreeArtifact() { + return objectFileTreeArtifact; + } + + @Override + public ActionOwner getOwner() { + return actionOwner; + } + + @Override + public boolean isShareable() { + return false; + } + + @Override + public String getMnemonic() { + return "LtoBackendActionTemplate"; + } + + @Override + public NestedSet getMandatoryInputs() { + return mandatoryInputs; + } + + @Override + public NestedSet getInputFilesForExtraAction( + ActionExecutionContext actionExecutionContext) { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + @Override + public ImmutableSet getMandatoryOutputs() { + return ImmutableSet.of(); + } + + @Override + public NestedSet getTools() { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + @Override + public NestedSet getInputs() { + return allInputs; + } + + @Override + public ImmutableSet getOutputs() { + ImmutableSet.Builder builder = ImmutableSet.builder(); + builder.add(objectFileTreeArtifact); + if (dwoFileTreeArtifact != null) { + builder.add(dwoFileTreeArtifact); + } + return builder.build(); + } + + @Override + public ImmutableList getClientEnvironmentVariables() { + return ImmutableList.of(); + } + + @Override + public NestedSet getSchedulingDependencies() { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + @Override + public boolean shouldReportPathPrefixConflict(ActionAnalysisMetadata action) { + return this != action; + } + + @Override + public MiddlemanType getActionType() { + return MiddlemanType.NORMAL; + } + + @Override + public String prettyPrint() { + return "LtoBackendActionTemplate compiling " + fullBitcodeTreeArtifact.getExecPathString(); + } + + @Override + public String describe() { + return "Lto backend compiling all C++ files in " + fullBitcodeTreeArtifact.prettyPrint(); + } + + @Override + public String toString() { + return prettyPrint(); + } + + private static DetailedExitCode makeDetailedExitCode(String message) { + return DetailedExitCode.of( + FailureDetails.FailureDetail.newBuilder() + .setMessage(message) + .setExecution( + FailureDetails.Execution.newBuilder() + .setCode( + FailureDetails.Execution.Code + .PERSISTENT_ACTION_OUTPUT_DIRECTORY_CREATION_FAILURE)) + .build()); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendArtifacts.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendArtifacts.java index b602ab280ed945..337f9cdf1925d3 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendArtifacts.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendArtifacts.java @@ -14,12 +14,13 @@ package com.google.devtools.build.lib.rules.cpp; - import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander; +import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact; import com.google.devtools.build.lib.actions.CommandLine; import com.google.devtools.build.lib.actions.CommandLineExpansionException; import com.google.devtools.build.lib.analysis.RuleErrorConsumer; @@ -82,14 +83,19 @@ public final class LtoBackendArtifacts implements LtoBackendArtifactsApi userCompileFlags) throws RuleErrorException, InterruptedException { + boolean createSharedNonLto = allBitcodeFiles == null; this.bitcodeFile = bitcodeFile; - PathFragment obj = ltoOutputRootPrefix.getRelative(bitcodeFile.getExecPath()); - - objectFile = - linkArtifactFactory.create(actionConstructionContext, repositoryName, configuration, obj); - imports = - linkArtifactFactory.create( - actionConstructionContext, - repositoryName, - configuration, - FileSystemUtils.appendExtension(obj, ".imports")); - index = - linkArtifactFactory.create( - actionConstructionContext, - repositoryName, - configuration, - FileSystemUtils.appendExtension(obj, ".thinlto.bc")); - - scheduleLtoBackendAction( - thread, - ruleErrorConsumer, - buildOptions, - cppConfiguration, - actionConstructionContext, - repositoryName, - featureConfiguration, - ccToolchain, - fdoContext, - usePic, - generateDwo, - configuration, - linkArtifactFactory, - userCompileFlags, - allBitcodeFiles); - } + PathFragment obj = ltoObjRootPrefix.getRelative(bitcodeFile.getExecPath()); + // indexObj is an object that does not exist but helps us find where to store the index and + // imports files + PathFragment indexObj = ltoOutputRootPrefix.getRelative(bitcodeFile.getExecPath()); - // Interface to create an LTO backend that does not perform any cross-module optimization. - public LtoBackendArtifacts( - StarlarkThread thread, - RuleErrorConsumer ruleErrorConsumer, - BuildOptions buildOptions, - CppConfiguration cppConfiguration, - PathFragment ltoOutputRootPrefix, - Artifact bitcodeFile, - ActionConstructionContext actionConstructionContext, - RepositoryName repositoryName, - BuildConfigurationValue configuration, - LinkArtifactFactory linkArtifactFactory, - FeatureConfiguration featureConfiguration, - CcToolchainProvider ccToolchain, - FdoContext fdoContext, - boolean usePic, - boolean generateDwo, - List userCompileFlags) - throws RuleErrorException, InterruptedException { - this.bitcodeFile = bitcodeFile; + LtoBackendAction.Builder builder = new LtoBackendAction.Builder(); - PathFragment obj = ltoOutputRootPrefix.getRelative(bitcodeFile.getExecPath()); - objectFile = - linkArtifactFactory.create(actionConstructionContext, repositoryName, configuration, obj); - imports = null; - index = null; + CcToolchainVariables ccToolchainVariables; - scheduleLtoBackendAction( - thread, - ruleErrorConsumer, - buildOptions, - cppConfiguration, - actionConstructionContext, - repositoryName, - featureConfiguration, + try { + ccToolchainVariables = ccToolchain.getBuildVariables(thread, buildOptions, cppConfiguration); + } catch (EvalException e) { + throw new RuleErrorException(e.getMessage()); + } + + CcToolchainVariables.Builder buildVariablesBuilder = + CcToolchainVariables.builder(ccToolchainVariables); + + initializeLtoBackendBuilder( + builder, + buildVariablesBuilder, ccToolchain, + cppConfiguration, fdoContext, - usePic, - generateDwo, - configuration, - linkArtifactFactory, + featureConfiguration, userCompileFlags, - /* bitcodeFiles= */ null); + ruleErrorConsumer); + CcToolchainVariables buildVariables = buildVariablesBuilder.build(); + if (bitcodeFile.isTreeArtifact()) { + objectFile = + linkArtifactFactory.createTreeArtifact( + actionConstructionContext, repositoryName, configuration, obj); + if (createSharedNonLto) { + imports = null; + index = null; + } else { + imports = + linkArtifactFactory.createTreeArtifact( + actionConstructionContext, repositoryName, configuration, indexObj); + index = imports; + } + if (generateDwo) { + // No support for dwo files for tree artifacts at the moment. This should not throw an + // irrecoverable exception because we can still generate dwo files for the other artifacts. + // TODO(b/289089713): Add support for dwo files for tree artifacts. + dwoFile = null; + } + createLtoBackendActionTemplate( + actionConstructionContext, + featureConfiguration, + builder, + buildVariables, + usePic, + allBitcodeFiles); + } else { + objectFile = + linkArtifactFactory.create(actionConstructionContext, repositoryName, configuration, obj); + if (createSharedNonLto) { + imports = null; + index = null; + } else { + String importsExt = Iterables.getOnlyElement(CppFileTypes.LTO_IMPORTS_FILE.getExtensions()); + String indexExt = + Iterables.getOnlyElement(CppFileTypes.LTO_INDEXING_ANALYSIS_FILE.getExtensions()); + imports = + linkArtifactFactory.create( + actionConstructionContext, + repositoryName, + configuration, + FileSystemUtils.appendExtension(indexObj, importsExt)); + index = + linkArtifactFactory.create( + actionConstructionContext, + repositoryName, + configuration, + FileSystemUtils.appendExtension(indexObj, indexExt)); + } + if (generateDwo) { + dwoFile = + linkArtifactFactory.create( + actionConstructionContext, + repositoryName, + configuration, + FileSystemUtils.replaceExtension( + objectFile.getOutputDirRelativePath(configuration.isSiblingRepositoryLayout()), + ".dwo")); + } + scheduleLtoBackendAction( + builder, + buildVariables, + actionConstructionContext, + featureConfiguration, + usePic, + allBitcodeFiles); + } } public Artifact getObjectFile() { @@ -217,69 +241,22 @@ void addIndexingOutputs(ImmutableSet.Builder builder) { builder.add(index); } - private void scheduleLtoBackendAction( - StarlarkThread thread, - RuleErrorConsumer ruleErrorConsumer, - BuildOptions buildOptions, - CppConfiguration cppConfiguration, - ActionConstructionContext actionConstructionContext, - RepositoryName repositoryName, - FeatureConfiguration featureConfiguration, + /** + * Populate buildVariablesBuilder, and builder with data that is independent of what file is the + * input to the action. + */ + private static void initializeLtoBackendBuilder( + LtoBackendAction.Builder builder, + CcToolchainVariables.Builder buildVariablesBuilder, CcToolchainProvider ccToolchain, + CppConfiguration cppConfiguration, FdoContext fdoContext, - boolean usePic, - boolean generateDwo, - BuildConfigurationValue configuration, - LinkArtifactFactory linkArtifactFactory, + FeatureConfiguration featureConfiguration, List userCompileFlags, - @Nullable BitcodeFiles bitcodeFiles) - throws RuleErrorException, InterruptedException { - LtoBackendAction.Builder builder = new LtoBackendAction.Builder(); - - builder.addInput(bitcodeFile); - - Preconditions.checkState( - (index == null) == (imports == null), - "Either both or neither index and imports files should be null"); - if (imports != null) { - builder.addImportsInfo(bitcodeFiles, imports); - // Although the imports file is not used by the LTOBackendAction while the action is - // executing, it is needed during the input discovery phase, and we must list it as an input - // to the action in order for it to be preserved under --discard_orphaned_artifacts. - builder.addInput(imports); - } - if (index != null) { - builder.addInput(index); - } + RuleErrorConsumer ruleErrorConsumer) + throws RuleErrorException { builder.addTransitiveInputs(ccToolchain.getCompilerFiles()); - - builder.addOutput(objectFile); - - builder.setProgressMessage("LTO Backend Compile %s", objectFile.getExecPath()); builder.setMnemonic("CcLtoBackendCompile"); - - CcToolchainVariables ccToolchainVariables; - - try { - ccToolchainVariables = ccToolchain.getBuildVariables(thread, buildOptions, cppConfiguration); - } catch (EvalException e) { - throw new RuleErrorException(e.getMessage()); - } - - CcToolchainVariables.Builder buildVariablesBuilder = - CcToolchainVariables.builder(ccToolchainVariables); - if (index != null) { - buildVariablesBuilder.addStringVariable("thinlto_index", index.getExecPath().toString()); - } else { - // An empty input indicates not to perform cross-module optimization. - buildVariablesBuilder.addStringVariable("thinlto_index", "/dev/null"); - } - // The output from the LTO backend step is a native object file. - buildVariablesBuilder.addStringVariable( - "thinlto_output_object_file", objectFile.getExecPath().toString()); - // The input to the LTO backend step is the bitcode file. - buildVariablesBuilder.addStringVariable( - "thinlto_input_bitcode_file", bitcodeFile.getExecPath().toString()); addProfileForLtoBackend(builder, fdoContext, featureConfiguration, buildVariablesBuilder); // Add the context sensitive instrument path to the backend. if (featureConfiguration.isEnabled(CppRuleClasses.CS_FDO_INSTRUMENT)) { @@ -287,28 +264,9 @@ private void scheduleLtoBackendAction( CompileBuildVariables.CS_FDO_INSTRUMENT_PATH.getVariableName(), ccToolchain.getCSFdoInstrument()); } - - if (generateDwo) { - dwoFile = - linkArtifactFactory.create( - actionConstructionContext, - repositoryName, - configuration, - FileSystemUtils.replaceExtension( - objectFile.getOutputDirRelativePath(configuration.isSiblingRepositoryLayout()), - ".dwo")); - builder.addOutput(dwoFile); - buildVariablesBuilder.addStringVariable( - CompileBuildVariables.PER_OBJECT_DEBUG_INFO_FILE.getVariableName(), - dwoFile.getExecPathString()); - buildVariablesBuilder.addStringVariable( - CompileBuildVariables.IS_USING_FISSION.getVariableName(), ""); - } buildVariablesBuilder.addStringSequenceVariable( CompileBuildVariables.USER_COMPILE_FLAGS.getVariableName(), userCompileFlags); - CcToolchainVariables buildVariables = buildVariablesBuilder.build(); - if (cppConfiguration.useStandaloneLtoIndexingCommandLines()) { if (!featureConfiguration.actionIsConfigured(CppActionNames.LTO_BACKEND)) { throw ruleErrorConsumer.throwWithRuleError( @@ -323,7 +281,75 @@ private void scheduleLtoBackendAction( PathFragment compiler = ccToolchain.getToolPathFragment(Tool.GCC, ruleErrorConsumer); builder.setExecutable(compiler); } + } + + private static void addPathsToBuildVariablesBuilder( + CcToolchainVariables.Builder buildVariablesBuilder, + String indexPath, + String objectFilePath, + String dwoFilePath, + String bitcodeFilePath) { + // Ideally, those strings would come directly from the execPath of the Artifacts of + // the LtoBackendAction.Builder; however, in order to support tree artifacts, we need + // the bitcodeFilePath to be different from the bitcodeTreeArtifact execPath. + // The former is a file path and the latter is the directory path. + // Therefore we accept strings as inputs rather than artifacts. + if (indexPath != null) { + buildVariablesBuilder.addStringVariable("thinlto_index", indexPath); + } else { + // An empty input indicates not to perform cross-module optimization. + buildVariablesBuilder.addStringVariable("thinlto_index", "/dev/null"); + } + // The output from the LTO backend step is a native object file. + buildVariablesBuilder.addStringVariable("thinlto_output_object_file", objectFilePath); + // The input to the LTO backend step is the bitcode file. + buildVariablesBuilder.addStringVariable("thinlto_input_bitcode_file", bitcodeFilePath); + // Add the context sensitive instrument path to the backend. + + if (dwoFilePath != null) { + buildVariablesBuilder.addStringVariable( + CompileBuildVariables.PER_OBJECT_DEBUG_INFO_FILE.getVariableName(), dwoFilePath); + buildVariablesBuilder.addStringVariable( + CompileBuildVariables.IS_USING_FISSION.getVariableName(), ""); + } + } + private static void addInputsToLtoBackendActionBuilder( + LtoBackendAction.Builder builder, + @Nullable Artifact index, + @Nullable Artifact imports, + Artifact bitcodeFile, + @Nullable BitcodeFiles bitcodeFiles) { + builder.addInput(bitcodeFile); + Preconditions.checkState( + (index == null) == (imports == null) && (imports == null) == (bitcodeFiles == null), + "Either all or none of index, imports and bitcodeFiles should be null"); + if (imports != null) { + builder.addImportsInfo(bitcodeFiles, imports); + // Although the imports file is not used by the LTOBackendAction while the action is + // executing, it is needed during the input discovery phase, and we must list it as an input + // to the action in order for it to be preserved under --discard_orphaned_artifacts. + builder.addInput(imports); + } + if (index != null) { + builder.addInput(index); + } + } + + private static void addOutputsToLtoBackendActionBuilder( + LtoBackendAction.Builder builder, Artifact objectFile, Artifact dwoFile) { + builder.addOutput(objectFile); + // Add the context sensitive instrument path to the backend. + if (dwoFile != null) { + builder.addOutput(dwoFile); + } + } + + private static void addCommandLineToLtoBackendActionBuilder( + LtoBackendAction.Builder builder, + FeatureConfiguration featureConfiguration, + CcToolchainVariables buildVariables, + boolean usePic) { CommandLine ltoCommandLine = new CommandLine() { @@ -354,6 +380,124 @@ public Iterable arguments(ArtifactExpander artifactExpander) } }; builder.addCommandLine(ltoCommandLine); + } + + /** + * Adds artifact to builder. The resulting builder can be built into a valid ltoBackendAction. + * + *

      Assumes that build and builderVariableBuilder have been initialized by calling {@link + * initializeLtoBackendBuilder}. If this is not true, the action will be wrong. + * + * @param builder the builder to add the artifacts to, initialized by initializeLtoBackendBuilder. + * @param buildVariables CcToolchainVariables initialized by initializeLtoBackendBuilder + * @param featureConfiguration the feature configuration to get the command line for the builder. + * @param index the index artifact to add. Can be a TreeFileArtifact but cannot be a Tree + * Artifact. + * @param imports the imports artifact to add. Can be a TreeFileArtifact but cannot be a Tree + * Artifact. + * @param bitcodeArtifact the bitcode artifact to add. If it is a Tree Artifact, bitcodeFilePath + * must be set. + * @param objectFile the object file to add. Can be a TreeFileArtifact but cannot be a Tree + * Artifact. + * @param bitcodeFiles the bitcode files to add. + * @param dwoFile the dwo file to add. + * @param usePic whether to add the PIC option to the command line. + * @param bitcodeFilePath the path of the bitcode object we are compiling. Only used if + * bitcodeArtifact is a tree artifact. + * @param isDummyAction if true then ignores the preconditions, because it is generating a dummy + * action, not a valid action. + */ + public static void addArtifactsLtoBackendAction( + LtoBackendAction.Builder builder, + CcToolchainVariables buildVariables, + FeatureConfiguration featureConfiguration, + @Nullable Artifact index, + @Nullable Artifact imports, + Artifact bitcodeArtifact, + Artifact objectFile, + @Nullable BitcodeFiles bitcodeFiles, + @Nullable Artifact dwoFile, + boolean usePic, + @Nullable String bitcodeFilePath, + boolean isDummyAction) { + Preconditions.checkState( + isDummyAction + || ((index == null || !index.isTreeArtifact()) + && (imports == null || !imports.isTreeArtifact()) + && (dwoFile == null || !dwoFile.isTreeArtifact()) + && !objectFile.isTreeArtifact()), + "index, imports, object and dwo files cannot be TreeArtifacts. We need to know their exact" + + " path not just directory path."); + Preconditions.checkState( + isDummyAction || (bitcodeArtifact.isTreeArtifact() ^ bitcodeFilePath == null), + "If bitcode file is a tree artifact, the bitcode file path must contain the path. If it is" + + " not a tree artifact, then bitcode file path should be null to not override the" + + " path."); + CcToolchainVariables.Builder buildVariablesBuilder = + CcToolchainVariables.builder(buildVariables); + addInputsToLtoBackendActionBuilder(builder, index, imports, bitcodeArtifact, bitcodeFiles); + addOutputsToLtoBackendActionBuilder(builder, objectFile, dwoFile); + builder.setProgressMessage("LTO Backend Compile %{output}"); + + String indexPath = index == null ? null : index.getExecPathString(); + String dwoFilePath = dwoFile == null ? null : dwoFile.getExecPathString(); + addPathsToBuildVariablesBuilder( + buildVariablesBuilder, + indexPath, + objectFile.getExecPathString(), + dwoFilePath, + bitcodeFilePath != null ? bitcodeFilePath : bitcodeArtifact.getExecPathString()); + CcToolchainVariables buildVariablesWithFiles = buildVariablesBuilder.build(); + addCommandLineToLtoBackendActionBuilder( + builder, featureConfiguration, buildVariablesWithFiles, usePic); + } + + private void createLtoBackendActionTemplate( + ActionConstructionContext actionConstructionContext, + FeatureConfiguration featureConfiguration, + LtoBackendAction.Builder ltoBackendActionbuilder, + CcToolchainVariables buildVariables, + boolean usePic, + BitcodeFiles bitcodeFiles) { + Preconditions.checkState( + (index == null && imports == null) || index.equals(imports), + "index and imports tree artifact must be the same"); + LtoBackendActionTemplate actionTemplate = + new LtoBackendActionTemplate( + (SpecialArtifact) index, + (SpecialArtifact) bitcodeFile, + (SpecialArtifact) objectFile, + (SpecialArtifact) dwoFile, + featureConfiguration, + ltoBackendActionbuilder, + buildVariables, + usePic, + bitcodeFiles, + actionConstructionContext.getActionOwner()); + actionConstructionContext.registerAction(actionTemplate); + } + + private void scheduleLtoBackendAction( + LtoBackendAction.Builder builder, + CcToolchainVariables buildVariables, + ActionConstructionContext actionConstructionContext, + FeatureConfiguration featureConfiguration, + boolean usePic, + @Nullable BitcodeFiles bitcodeFiles) { + + addArtifactsLtoBackendAction( + builder, + buildVariables, + featureConfiguration, + index, + imports, + bitcodeFile, + objectFile, + bitcodeFiles, + dwoFile, + usePic, + /* bitcodeFilePath= */ null, + /* isDummyAction= */ false); actionConstructionContext.registerAction(builder.build(actionConstructionContext)); } diff --git a/src/main/java/com/google/devtools/build/lib/rules/nativedeps/NativeDepsHelper.java b/src/main/java/com/google/devtools/build/lib/rules/nativedeps/NativeDepsHelper.java index e9f786e31a9e02..f267c2ddc21923 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/nativedeps/NativeDepsHelper.java +++ b/src/main/java/com/google/devtools/build/lib/rules/nativedeps/NativeDepsHelper.java @@ -27,10 +27,8 @@ import com.google.devtools.build.lib.actions.ArtifactRoot; import com.google.devtools.build.lib.analysis.AnalysisUtils; import com.google.devtools.build.lib.analysis.RuleContext; -import com.google.devtools.build.lib.analysis.actions.ActionConstructionContext; import com.google.devtools.build.lib.analysis.actions.SymlinkAction; import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; -import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; import com.google.devtools.build.lib.packages.TargetUtils; @@ -71,26 +69,6 @@ * that some rules are implicitly neverlink. */ public abstract class NativeDepsHelper { - /** - * An implementation of {@link - * com.google.devtools.build.lib.rules.cpp.CppLinkAction.LinkArtifactFactory} that can create - * artifacts anywhere. - * - *

      Necessary because the actions of nativedeps libraries should be shareable, and thus cannot - * be under the package directory. - */ - private static final CppLinkAction.LinkArtifactFactory SHAREABLE_LINK_ARTIFACT_FACTORY = - new CppLinkAction.LinkArtifactFactory() { - @Override - public Artifact create( - ActionConstructionContext actionConstructionContext, - RepositoryName repositoryName, - BuildConfigurationValue configuration, - PathFragment rootRelativePath) { - return actionConstructionContext.getShareableArtifact( - rootRelativePath, configuration.getBinDirectory(repositoryName)); - } - }; private NativeDepsHelper() {} @@ -288,7 +266,7 @@ public static NativeDepsRunfiles createNativeDepsAction( .setNeverLink(true) .setShouldCreateStaticLibraries(false) .addCcLinkingContexts(ImmutableList.of(ccLinkingContext)) - .setLinkArtifactFactory(SHAREABLE_LINK_ARTIFACT_FACTORY) + .setLinkArtifactFactory(CppLinkAction.SHAREABLE_LINK_ARTIFACT_FACTORY) .setDynamicLinkType(LinkTargetType.DYNAMIC_LIBRARY) .link(CcCompilationOutputs.EMPTY); diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/AppleBinary.java b/src/main/java/com/google/devtools/build/lib/rules/objc/AppleBinary.java index b7f8f8c481b377..7ef86f865543ec 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/objc/AppleBinary.java +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/AppleBinary.java @@ -20,7 +20,6 @@ import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException; import com.google.devtools.build.lib.analysis.OutputGroupInfo; import com.google.devtools.build.lib.analysis.RuleContext; import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; @@ -39,6 +38,7 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; +import net.starlark.java.eval.EvalException; /** Native support for Apple binary rules. */ public class AppleBinary { @@ -70,7 +70,7 @@ public static AppleLinkingOutputs linkMultiArchBinary( Iterable extraRequestedFeatures, Iterable extraDisabledFeatures, boolean isStampingEnabled) - throws InterruptedException, RuleErrorException, ActionConflictException { + throws InterruptedException, RuleErrorException, EvalException { Map, List> splitDeps = ruleContext.getSplitPrerequisiteConfiguredTargetAndTargets("deps"); Map, List> splitToolchains = diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/AppleStarlarkCommon.java b/src/main/java/com/google/devtools/build/lib/rules/objc/AppleStarlarkCommon.java index d1da07021e1892..2e2e1c1da626c1 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/objc/AppleStarlarkCommon.java +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/AppleStarlarkCommon.java @@ -19,7 +19,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException; import com.google.devtools.build.lib.analysis.RuleContext; import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; @@ -283,7 +282,7 @@ public StructImpl linkMultiArchBinary( Sequence.cast(extraDisabledFeatures, String.class, "extra_disabled_features"), isStampingEnabled); return createStarlarkLinkingOutputs(linkingOutputs, thread); - } catch (RuleErrorException | ActionConflictException exception) { + } catch (RuleErrorException exception) { throw new EvalException(exception); } } diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java index 0286cdbf8c1378..c31af792f651b5 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java @@ -42,7 +42,6 @@ import com.google.devtools.build.lib.actions.CommandLine; import com.google.devtools.build.lib.actions.ParamFileInfo; import com.google.devtools.build.lib.actions.ParameterFile; -import com.google.devtools.build.lib.analysis.FilesToRunProvider; import com.google.devtools.build.lib.analysis.RuleContext; import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; import com.google.devtools.build.lib.analysis.actions.CustomCommandLine.VectorArg; @@ -50,11 +49,13 @@ import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; +import com.google.devtools.build.lib.collect.nestedset.Depset; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; import com.google.devtools.build.lib.collect.nestedset.Order; import com.google.devtools.build.lib.packages.BuildType; import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; +import com.google.devtools.build.lib.packages.StarlarkInfo; import com.google.devtools.build.lib.packages.TargetUtils; import com.google.devtools.build.lib.rules.apple.AppleConfiguration; import com.google.devtools.build.lib.rules.apple.XcodeConfigInfo; @@ -139,11 +140,6 @@ public class CompilationSupport implements StarlarkValue { private static final Predicate ALWAYS_LINKED_CC_LIBRARY = input -> LINK_LIBRARY_FILETYPES.matches(input.getFilename()); - /** Returns the location of the xcrunwrapper tool. */ - public static final FilesToRunProvider xcrunwrapper(RuleContext ruleContext) { - return ruleContext.getExecutablePrerequisite("$xcrunwrapper"); - } - /** Iterable wrapper providing strong type safety for arguments to binary linking. */ static final class ExtraLinkArgs extends IterableWrapper { ExtraLinkArgs(String... args) { @@ -536,14 +532,14 @@ public CompilationSupport registerLinkActions( Object linkingInfoProvider, ObjcProvider secondaryObjcProvider, CcLinkingContext secondaryCcLinkingContext, - J2ObjcMappingFileProvider j2ObjcMappingFileProvider, - J2ObjcEntryClassProvider j2ObjcEntryClassProvider, + StarlarkInfo j2ObjcMappingFileProvider, + StarlarkInfo j2ObjcEntryClassProvider, ExtraLinkArgs extraLinkArgs, Iterable extraLinkInputs, Iterable extraRequestedFeatures, Iterable extraDisabledFeatures, boolean isStampingEnabled) - throws InterruptedException, RuleErrorException { + throws InterruptedException, RuleErrorException, EvalException { ObjcProvider objcProviderWithLinkingInfo = null; CcLinkingContext ccLinkingContextWithLinkingInfo = null; checkState( @@ -824,30 +820,39 @@ private static ImmutableSet getForceLoadArtifacts(ObjcProvider objcPro .build(); } + private NestedSet getField(StarlarkInfo provider, String fieldName, Class type) + throws EvalException { + return Depset.cast(provider.getValue(fieldName), type, fieldName); + } + /** Returns true if this build should strip J2Objc dead code. */ - private boolean stripJ2ObjcDeadCode(J2ObjcEntryClassProvider j2ObjcEntryClassProvider) { + private boolean stripJ2ObjcDeadCode(StarlarkInfo j2ObjcEntryClassProvider) throws EvalException { J2ObjcConfiguration j2objcConfiguration = buildConfiguration.getFragment(J2ObjcConfiguration.class); + NestedSet entryClasses = + getField(j2ObjcEntryClassProvider, "entry_classes", String.class); + // Only perform J2ObjC dead code stripping if flag --j2objc_dead_code_removal is specified and // users have specified entry classes. - return j2objcConfiguration.removeDeadCode() - && !j2ObjcEntryClassProvider.getEntryClasses().isEmpty(); + return j2objcConfiguration.removeDeadCode() && !entryClasses.isEmpty(); } /** Registers actions to perform J2Objc dead code removal. */ private void registerJ2ObjcDeadCodeRemovalActions( ObjcProvider objcProvider, - J2ObjcMappingFileProvider j2ObjcMappingFileProvider, - J2ObjcEntryClassProvider j2ObjcEntryClassProvider) { + StarlarkInfo j2ObjcMappingFileProvider, + StarlarkInfo j2ObjcEntryClassProvider) + throws EvalException { ObjcConfiguration objcConfiguration = buildConfiguration.getFragment(ObjcConfiguration.class); - NestedSet entryClasses = j2ObjcEntryClassProvider.getEntryClasses(); + NestedSet entryClasses = + getField(j2ObjcEntryClassProvider, "entry_classes", String.class); NestedSet j2ObjcDependencyMappingFiles = - j2ObjcMappingFileProvider.getDependencyMappingFiles(); + getField(j2ObjcMappingFileProvider, "dependency_mapping_files", Artifact.class); NestedSet j2ObjcHeaderMappingFiles = - j2ObjcMappingFileProvider.getHeaderMappingFiles(); + getField(j2ObjcMappingFileProvider, "header_mapping_files", Artifact.class); NestedSet j2ObjcArchiveSourceMappingFiles = - j2ObjcMappingFileProvider.getArchiveSourceMappingFiles(); + getField(j2ObjcMappingFileProvider, "archive_source_mapping_files", Artifact.class); for (Artifact j2objcArchive : objcProvider.get(ObjcProvider.J2OBJC_LIBRARY).toList()) { Artifact prunedJ2ObjcArchive = intermediateArtifacts.j2objcPrunedArchive(j2objcArchive); @@ -873,7 +878,6 @@ private void registerJ2ObjcDeadCodeRemovalActions( .addExecPath("--input_archive", j2objcArchive) .addExecPath("--output_archive", prunedJ2ObjcArchive) .addExecPath("--dummy_archive", dummyArchive) - .addExecPath("--xcrunwrapper", xcrunwrapper(ruleContext).getExecutable()) .addExecPaths( "--dependency_mapping_files", VectorArg.join(",").each(j2ObjcDependencyMappingFiles)) @@ -894,7 +898,6 @@ private void registerJ2ObjcDeadCodeRemovalActions( .setExecutable(ruleContext.getExecutablePrerequisite("$j2objc_dead_code_pruner")) .addInput(dummyArchive) .addInput(j2objcArchive) - .addInput(xcrunwrapper(ruleContext).getExecutable()) .addTransitiveInputs(j2ObjcDependencyMappingFiles) .addTransitiveInputs(j2ObjcHeaderMappingFiles) .addTransitiveInputs(j2ObjcArchiveSourceMappingFiles) @@ -944,6 +947,7 @@ private Artifact getBinaryToLink() { private static CommandLine symbolStripCommandLine( ImmutableList extraFlags, Artifact unstrippedArtifact, Artifact strippedArtifact) { return CustomCommandLine.builder() + .add("/usr/bin/xcrun") .add(STRIP) .addAll(extraFlags) .addExecPath("-o", strippedArtifact) @@ -986,7 +990,6 @@ private void registerBinaryStripAction(Artifact binaryToLink, StrippingType stri XcodeConfigInfo.fromRuleContext(ruleContext), appleConfiguration.getSingleArchPlatform()) .setMnemonic("ObjcBinarySymbolStrip") - .setExecutable(xcrunwrapper(ruleContext)) .addCommandLine(symbolStripCommandLine(stripArgs, binaryToLink, strippedBinary)) .addOutput(strippedBinary) .addInput(binaryToLink) diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcAspect.java b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcAspect.java deleted file mode 100644 index 6045ba019edaaf..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcAspect.java +++ /dev/null @@ -1,962 +0,0 @@ -// Copyright 2015 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.rules.objc; - -import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.google.devtools.build.lib.packages.Attribute.attr; -import static com.google.devtools.build.lib.packages.BuildType.LABEL; -import static java.nio.charset.StandardCharsets.ISO_8859_1; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException; -import com.google.devtools.build.lib.actions.ParamFileInfo; -import com.google.devtools.build.lib.actions.ParameterFile; -import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; -import com.google.devtools.build.lib.analysis.ConfiguredAspect; -import com.google.devtools.build.lib.analysis.ConfiguredAspectFactory; -import com.google.devtools.build.lib.analysis.ConfiguredTarget; -import com.google.devtools.build.lib.analysis.RuleContext; -import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; -import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; -import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; -import com.google.devtools.build.lib.analysis.actions.CustomCommandLine.VectorArg; -import com.google.devtools.build.lib.analysis.actions.SpawnAction; -import com.google.devtools.build.lib.analysis.config.ConfigAwareAspectBuilder; -import com.google.devtools.build.lib.analysis.config.ExecutionTransitionFactory; -import com.google.devtools.build.lib.analysis.config.ToolchainTypeRequirement; -import com.google.devtools.build.lib.analysis.platform.ToolchainInfo; -import com.google.devtools.build.lib.cmdline.Label; -import com.google.devtools.build.lib.cmdline.RepositoryName; -import com.google.devtools.build.lib.collect.nestedset.NestedSet; -import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; -import com.google.devtools.build.lib.packages.AspectDefinition; -import com.google.devtools.build.lib.packages.AspectParameters; -import com.google.devtools.build.lib.packages.Attribute.LabelLateBoundDefault; -import com.google.devtools.build.lib.packages.Attribute.LateBoundDefault.Resolver; -import com.google.devtools.build.lib.packages.BuildType; -import com.google.devtools.build.lib.packages.ExecGroup; -import com.google.devtools.build.lib.packages.NativeAspectClass; -import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; -import com.google.devtools.build.lib.packages.StarlarkInfo; -import com.google.devtools.build.lib.packages.StarlarkProviderIdentifier; -import com.google.devtools.build.lib.rules.apple.AppleConfiguration; -import com.google.devtools.build.lib.rules.apple.AppleToolchain; -import com.google.devtools.build.lib.rules.apple.XcodeConfigRule; -import com.google.devtools.build.lib.rules.cpp.CcCompilationContext; -import com.google.devtools.build.lib.rules.cpp.CcInfo; -import com.google.devtools.build.lib.rules.cpp.CcLinkingContext; -import com.google.devtools.build.lib.rules.cpp.CcToolchain; -import com.google.devtools.build.lib.rules.cpp.CcToolchainProvider; -import com.google.devtools.build.lib.rules.cpp.CppConfiguration; -import com.google.devtools.build.lib.rules.cpp.CppHelper; -import com.google.devtools.build.lib.rules.cpp.CppModuleMap.UmbrellaHeaderStrategy; -import com.google.devtools.build.lib.rules.cpp.CppRuleClasses; -import com.google.devtools.build.lib.rules.cpp.CppSemantics; -import com.google.devtools.build.lib.rules.java.JavaInfo; -import com.google.devtools.build.lib.rules.java.JavaRuleClasses; -import com.google.devtools.build.lib.rules.java.JavaSemantics; -import com.google.devtools.build.lib.rules.java.JavaToolchainProvider; -import com.google.devtools.build.lib.rules.objc.IntermediateArtifacts.AlwaysLink; -import com.google.devtools.build.lib.rules.objc.J2ObjcSource.SourceType; -import com.google.devtools.build.lib.rules.proto.ProtoCommon; -import com.google.devtools.build.lib.rules.proto.ProtoConfiguration; -import com.google.devtools.build.lib.rules.proto.ProtoInfo; -import com.google.devtools.build.lib.rules.proto.ProtoLangToolchainProvider; -import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; -import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; -import com.google.devtools.build.lib.vfs.PathFragment; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import javax.annotation.Nullable; -import net.starlark.java.eval.EvalException; -import net.starlark.java.eval.StarlarkList; -import net.starlark.java.eval.Tuple; - -/** J2ObjC transpilation aspect for Java and proto rules. */ -public class J2ObjcAspect extends NativeAspectClass implements ConfiguredAspectFactory { - public static final String NAME = "J2ObjcAspect"; - - private static LabelLateBoundDefault getProtoToolchainLabel( - String defaultValue) { - return LabelLateBoundDefault.fromTargetConfiguration( - ProtoConfiguration.class, - Label.parseCanonicalUnchecked(defaultValue), - (Resolver & Serializable) - (rule, attributes, protoConfig) -> protoConfig.protoToolchainForJ2objc()); - } - - private static final ImmutableList JAVA_DEPENDENT_ATTRIBUTES = - ImmutableList.of("$jre_lib", "deps", "exports", "runtime_deps"); - - private static final ImmutableList PROTO_DEPENDENT_ATTRIBUTES = ImmutableList.of("deps"); - - private static final String J2OBJC_PROTO_TOOLCHAIN_ATTR = ":j2objc_proto_toolchain"; - - @SerializationConstant @AutoCodec.VisibleForSerialization - static final LabelLateBoundDefault DEAD_CODE_REPORT = - LabelLateBoundDefault.fromTargetConfiguration( - J2ObjcConfiguration.class, - null, - (rule, attributes, j2objcConfig) -> j2objcConfig.deadCodeReport()); - - private final RepositoryName toolsRepository; - private final Label ccToolchainType; - private final LabelLateBoundDefault ccToolchain; - private final ToolchainTypeRequirement javaToolchainTypeRequirement; - - public J2ObjcAspect(RuleDefinitionEnvironment env, CppSemantics cppSemantics) { - this.toolsRepository = checkNotNull(env.getToolsRepository()); - this.ccToolchainType = CppRuleClasses.ccToolchainTypeAttribute(env); - this.ccToolchain = CppRuleClasses.ccToolchainAttribute(env); - this.javaToolchainTypeRequirement = JavaRuleClasses.javaToolchainTypeRequirement(env); - } - - @Override - public AspectDefinition getDefinition(AspectParameters aspectParameters) { - return ConfigAwareAspectBuilder.of(new AspectDefinition.Builder(this)) - .originalBuilder() - .propagateAlongAttribute("deps") - .propagateAlongAttribute("exports") - .propagateAlongAttribute("runtime_deps") - .requireStarlarkProviders(StarlarkProviderIdentifier.forKey(JavaInfo.PROVIDER.getKey())) - .requireStarlarkProviders(ProtoInfo.PROVIDER.id()) - .advertiseProvider(ImmutableList.of(ObjcProvider.STARLARK_CONSTRUCTOR.id())) - .requiresConfigurationFragments( - AppleConfiguration.class, - CppConfiguration.class, - J2ObjcConfiguration.class, - ObjcConfiguration.class, - ProtoConfiguration.class) - .addToolchainTypes(CppRuleClasses.ccToolchainTypeRequirement(ccToolchainType)) - .add( - attr("$grep_includes", LABEL) - .cfg(ExecutionTransitionFactory.createFactory()) - .value( - Label.parseCanonicalUnchecked(toolsRepository + "//tools/cpp:grep-includes"))) - .add( - attr("$j2objc", LABEL) - .cfg(ExecutionTransitionFactory.createFactory("j2objc")) - .exec() - .value( - Label.parseCanonicalUnchecked( - toolsRepository + "//tools/j2objc:j2objc_deploy.jar"))) - .add( - attr("$j2objc_wrapper", LABEL) - .cfg(ExecutionTransitionFactory.createFactory("j2objc")) - .exec() - .legacyAllowAnyFileType() - .value( - Label.parseCanonicalUnchecked( - toolsRepository + "//tools/j2objc:j2objc_wrapper_binary"))) - .add( - attr("$j2objc_header_map", LABEL) - .cfg(ExecutionTransitionFactory.createFactory("j2objc")) - .exec() - .legacyAllowAnyFileType() - .value( - Label.parseCanonicalUnchecked( - toolsRepository + "//tools/j2objc:j2objc_header_map_binary"))) - .add( - attr("$jre_emul_jar", LABEL) - .cfg(ExecutionTransitionFactory.createFactory("j2objc")) - .value( - Label.parseCanonicalUnchecked( - toolsRepository + "//third_party/java/j2objc:jre_emul.jar"))) - .add( - attr("$jre_emul_module", LABEL) - .cfg(ExecutionTransitionFactory.createFactory("j2objc")) - .value( - Label.parseCanonicalUnchecked( - toolsRepository + "//third_party/java/j2objc:jre_emul_module"))) - .add( - attr(":dead_code_report", LABEL) - .cfg(ExecutionTransitionFactory.createFactory("j2objc")) - .value(DEAD_CODE_REPORT)) - .add( - attr("$jre_lib", LABEL) - .value( - Label.parseCanonicalUnchecked( - toolsRepository + "//third_party/java/j2objc:jre_core_lib"))) - .add( - attr("$xcrunwrapper", LABEL) - .cfg(ExecutionTransitionFactory.createFactory()) - .exec() - .value( - Label.parseCanonicalUnchecked(toolsRepository + "//tools/objc:xcrunwrapper"))) - .add( - attr(XcodeConfigRule.XCODE_CONFIG_ATTR_NAME, LABEL) - .allowedRuleClasses("xcode_config") - .checkConstraints() - .value(AppleToolchain.getXcodeConfigLabel(toolsRepository))) - .add( - attr("$zipper", LABEL) - .cfg(ExecutionTransitionFactory.createFactory()) - .exec() - .value(Label.parseCanonicalUnchecked(toolsRepository + "//tools/zip:zipper"))) - .add( - attr(J2OBJC_PROTO_TOOLCHAIN_ATTR, LABEL) - .legacyAllowAnyFileType() - .value( - getProtoToolchainLabel( - toolsRepository + "//tools/j2objc:j2objc_proto_toolchain"))) - .add( - attr(JavaRuleClasses.JAVA_TOOLCHAIN_TYPE_ATTRIBUTE_NAME, LABEL) - .value(javaToolchainTypeRequirement.toolchainType())) - .add( - attr(CcToolchain.CC_TOOLCHAIN_DEFAULT_ATTRIBUTE_NAME, LABEL) - .mandatoryProviders(CcToolchainProvider.PROVIDER.id()) - .value(ccToolchain)) - .execGroups( - ImmutableMap.of( - "proto_compiler", - ExecGroup.builder().build(), - "j2objc", - ExecGroup.builder().addToolchainType(javaToolchainTypeRequirement).build())) - .build(); - } - - @Override - public ConfiguredAspect create( - Label targetLabel, - ConfiguredTarget ct, - RuleContext ruleContext, - AspectParameters parameters, - RepositoryName toolsRepository) - throws InterruptedException, ActionConflictException { - if (isProtoRule(ct)) { - return proto(ct, ruleContext); - } - try { - return java(ct, ruleContext); - } catch (EvalException | RuleErrorException e) { - ruleContext.ruleError(e.getMessage()); - return null; - } - } - - /** - * Returns a {@link IntermediateArtifacts} to be used to compile and link the ObjC source files - * generated by J2ObjC. - */ - private static IntermediateArtifacts j2objcIntermediateArtifacts(RuleContext ruleContext) { - // We need to append "_j2objc" to the name of the generated archive file to distinguish it from - // the C/C++ archive file created by proto_library targets with attribute cc_api_version - // specified. - // Generate an umbrella header for the module map. The headers declared in module maps are - // compiled using the #import directives which are incompatible with J2ObjC segmented headers. - // We need to #iclude all the headers in an umbrella header and then declare the umbrella header - // in the module map. - return new IntermediateArtifacts( - ruleContext, - /* archiveFileNameSuffix= */ "_j2objc", - UmbrellaHeaderStrategy.GENERATE, - AlwaysLink.TRUE); - } - - private ConfiguredAspect buildAspect( - ConfiguredTarget base, - RuleContext ruleContext, - J2ObjcSource j2ObjcSource, - J2ObjcMappingFileProvider directJ2ObjcMappingFileProvider, - List depAttributes, - List otherDeps) - throws InterruptedException, ActionConflictException { - ConfiguredAspect.Builder builder = new ConfiguredAspect.Builder(ruleContext); - ObjcCommon common; - CcCompilationContext ccCompilationContext = null; - CcLinkingContext ccLinkingContext = null; - - IntermediateArtifacts intermediateArtifacts = j2objcIntermediateArtifacts(ruleContext); - if (!j2ObjcSource.getObjcSrcs().isEmpty()) { - common = - common( - ObjcCommon.Purpose.COMPILE_AND_LINK, - ruleContext, - intermediateArtifacts, - j2ObjcSource.getObjcSrcs(), - j2ObjcSource.getObjcHdrs(), - j2ObjcSource.getHeaderSearchPaths(), - depAttributes, - otherDeps); - - try { - CcToolchainProvider ccToolchain = - CppHelper.getToolchain( - ruleContext, - ruleContext.getPrerequisite(CcToolchain.CC_TOOLCHAIN_DEFAULT_ATTRIBUTE_NAME), - ccToolchainType); - ImmutableList extraCompileArgs = - j2objcCompileWithARC(ruleContext) - ? ImmutableList.of("-fno-strict-overflow", "-fobjc-arc-exceptions") - : ImmutableList.of("-fno-strict-overflow", "-fobjc-weak"); - - Object starlarkFunc = - ruleContext.getStarlarkDefinedBuiltin( - "register_compile_and_archive_actions_for_j2objc"); - ruleContext.initStarlarkRuleContext(); - Tuple compilationResult = - (Tuple) - ruleContext.callStarlarkOrThrowRuleError( - starlarkFunc, - ImmutableList.of( - ruleContext.getStarlarkRuleContext(), - ccToolchain, - intermediateArtifacts, - common.getCompilationArtifacts().get(), - common.getObjcCompilationContext(), - StarlarkList.immutableCopyOf(common.getCcLinkingContexts()), - StarlarkList.immutableCopyOf(extraCompileArgs)), - new HashMap<>()); - - ccCompilationContext = (CcCompilationContext) compilationResult.get(0); - ccLinkingContext = (CcLinkingContext) compilationResult.get(1); - } catch (RuleErrorException e) { - ruleContext.ruleError(e.getMessage()); - } - } else { - common = - common( - ObjcCommon.Purpose.LINK_ONLY, - ruleContext, - intermediateArtifacts, - ImmutableList.of(), - ImmutableList.of(), - ImmutableList.of(), - depAttributes, - otherDeps); - ccCompilationContext = common.createCcCompilationContext(); - ccLinkingContext = common.createCcLinkingContext(); - } - - return builder - .addNativeDeclaredProvider( - exportedJ2ObjcMappingFileProvider(base, ruleContext, directJ2ObjcMappingFileProvider)) - .addNativeDeclaredProvider(common.getObjcProvider()) - .addNativeDeclaredProvider( - CcInfo.builder() - .setCcCompilationContext(ccCompilationContext) - .setCcLinkingContext(ccLinkingContext) - .build()) - .build(); - } - - private ConfiguredAspect java(ConfiguredTarget base, RuleContext ruleContext) - throws InterruptedException, ActionConflictException, EvalException, RuleErrorException { - NestedSet compileTimeJars = JavaInfo.transitiveCompileTimeJars(base); - ImmutableSet.Builder javaSourceFilesBuilder = ImmutableSet.builder(); - ImmutableSet.Builder javaSourceJarsBuilder = ImmutableSet.builder(); - - for (Artifact srcArtifact : ruleContext.getPrerequisiteArtifacts("srcs").list()) { - String srcFilename = srcArtifact.getExecPathString(); - if (JavaSemantics.SOURCE_JAR.apply(srcFilename)) { - javaSourceJarsBuilder.add(srcArtifact); - } else if (JavaSemantics.JAVA_SOURCE.apply(srcFilename)) { - javaSourceFilesBuilder.add(srcArtifact); - } - } - Artifact srcJar = - ruleContext.attributes().has("srcjar") - ? ruleContext.getPrerequisiteArtifact("srcjar") - : null; - if (srcJar != null) { - javaSourceJarsBuilder.add(srcJar); - } - - JavaInfo.genSourceJar(base).ifPresent(javaSourceJarsBuilder::add); - - ImmutableList javaSourceFiles = javaSourceFilesBuilder.build().asList(); - ImmutableList javaSourceJars = javaSourceJarsBuilder.build().asList(); - J2ObjcSource j2ObjcSource = javaJ2ObjcSource(ruleContext, javaSourceFiles, javaSourceJars); - J2ObjcMappingFileProvider depJ2ObjcMappingFileProvider = - depJ2ObjcMappingFileProvider(ruleContext); - - J2ObjcMappingFileProvider directJ2ObjcMappingFileProvider; - if (j2ObjcSource.getObjcSrcs().isEmpty()) { - directJ2ObjcMappingFileProvider = new J2ObjcMappingFileProvider.Builder().build(); - } else { - directJ2ObjcMappingFileProvider = - createJ2ObjcTranspilationAction( - ruleContext, - javaSourceFiles, - javaSourceJars, - depJ2ObjcMappingFileProvider, - compileTimeJars, - j2ObjcSource); - } - return buildAspect( - base, - ruleContext, - j2ObjcSource, - directJ2ObjcMappingFileProvider, - JAVA_DEPENDENT_ATTRIBUTES, - ImmutableList.of()); - } - - @Nullable - private ConfiguredAspect proto(ConfiguredTarget base, RuleContext ruleContext) - throws InterruptedException, ActionConflictException { - ProtoLangToolchainProvider protoToolchain = - ProtoLangToolchainProvider.get(ruleContext, J2OBJC_PROTO_TOOLCHAIN_ATTR); - StarlarkInfo starlarkProtoToolchain = - ProtoLangToolchainProvider.getStarlarkProvider(ruleContext, J2OBJC_PROTO_TOOLCHAIN_ATTR); - try { - // Avoid pulling in any generated files from forbidden protos. - ImmutableList filteredProtoSources = - ImmutableList.copyOf( - ProtoCommon.filterSources(ruleContext, base, starlarkProtoToolchain)); - - J2ObjcSource j2ObjcSource = protoJ2ObjcSource(ruleContext, base, filteredProtoSources); - - J2ObjcMappingFileProvider directJ2ObjcMappingFileProvider; - if (j2ObjcSource.getObjcSrcs().isEmpty()) { - directJ2ObjcMappingFileProvider = new J2ObjcMappingFileProvider.Builder().build(); - } else { - - directJ2ObjcMappingFileProvider = - createJ2ObjcProtoCompileActions( - base, starlarkProtoToolchain, ruleContext, filteredProtoSources, j2ObjcSource); - } - - return buildAspect( - base, - ruleContext, - j2ObjcSource, - directJ2ObjcMappingFileProvider, - PROTO_DEPENDENT_ATTRIBUTES, - ImmutableList.of(protoToolchain.runtime())); - } catch (RuleErrorException e) { - ruleContext.ruleError(e.getMessage()); - return null; - } - } - - private static J2ObjcMappingFileProvider exportedJ2ObjcMappingFileProvider( - ConfiguredTarget base, - RuleContext ruleContext, - J2ObjcMappingFileProvider directJ2ObjcMappingFileProvider) { - J2ObjcMappingFileProvider depJ2ObjcMappingFileProvider = - depJ2ObjcMappingFileProvider(ruleContext); - - NestedSetBuilder exportedHeaderMappingFiles = - NestedSetBuilder.stableOrder() - .addTransitive(directJ2ObjcMappingFileProvider.getHeaderMappingFiles()); - - NestedSetBuilder exportedClassMappingFiles = - NestedSetBuilder.stableOrder() - .addTransitive(directJ2ObjcMappingFileProvider.getClassMappingFiles()) - .addTransitive(depJ2ObjcMappingFileProvider.getClassMappingFiles()); - - NestedSetBuilder exportedDependencyMappingFiles = - NestedSetBuilder.stableOrder() - .addTransitive(directJ2ObjcMappingFileProvider.getDependencyMappingFiles()) - .addTransitive(depJ2ObjcMappingFileProvider.getDependencyMappingFiles()); - - NestedSetBuilder archiveSourceMappingFiles = - NestedSetBuilder.stableOrder() - .addTransitive(directJ2ObjcMappingFileProvider.getArchiveSourceMappingFiles()) - .addTransitive(depJ2ObjcMappingFileProvider.getArchiveSourceMappingFiles()); - - // J2ObjC merges all transitive input header mapping files into one header mapping file, - // so we only need to re-export other dependent output header mapping files in proto rules and - // rules where J2ObjC is not run (e.g., no sources). - // We also add the transitive header mapping files if experimental J2ObjC header mapping is - // turned on. The experimental support does not merge transitive input header mapping files. - boolean experimentalJ2ObjcHeaderMap = - ruleContext.getFragment(J2ObjcConfiguration.class).experimentalJ2ObjcHeaderMap(); - if (isProtoRule(base) || exportedHeaderMappingFiles.isEmpty() || experimentalJ2ObjcHeaderMap) { - exportedHeaderMappingFiles.addTransitive( - depJ2ObjcMappingFileProvider.getHeaderMappingFiles()); - } - - return new J2ObjcMappingFileProvider( - exportedHeaderMappingFiles.build(), - exportedClassMappingFiles.build(), - exportedDependencyMappingFiles.build(), - archiveSourceMappingFiles.build()); - } - - private static J2ObjcMappingFileProvider depJ2ObjcMappingFileProvider(RuleContext ruleContext) { - NestedSetBuilder depsHeaderMappingsBuilder = NestedSetBuilder.stableOrder(); - NestedSetBuilder depsClassMappingsBuilder = NestedSetBuilder.stableOrder(); - NestedSetBuilder depsDependencyMappingsBuilder = NestedSetBuilder.stableOrder(); - NestedSetBuilder depsArchiveSourceMappingsBuilder = NestedSetBuilder.stableOrder(); - - for (J2ObjcMappingFileProvider mapping : getJ2ObjCMappings(ruleContext)) { - depsHeaderMappingsBuilder.addTransitive(mapping.getHeaderMappingFiles()); - depsClassMappingsBuilder.addTransitive(mapping.getClassMappingFiles()); - depsDependencyMappingsBuilder.addTransitive(mapping.getDependencyMappingFiles()); - depsArchiveSourceMappingsBuilder.addTransitive(mapping.getArchiveSourceMappingFiles()); - } - - return new J2ObjcMappingFileProvider( - depsHeaderMappingsBuilder.build(), - depsClassMappingsBuilder.build(), - depsDependencyMappingsBuilder.build(), - depsArchiveSourceMappingsBuilder.build()); - } - - private static ImmutableList sourceJarFlags(RuleContext ruleContext) { - return ImmutableList.of( - "--output_gen_source_dir", - j2ObjcSourceJarTranslatedSourceTreeArtifact(ruleContext).getExecPathString(), - "--output_gen_header_dir", - j2objcSourceJarTranslatedHeaderTreeArtifact(ruleContext).getExecPathString()); - } - - private static J2ObjcMappingFileProvider createJ2ObjcTranspilationAction( - RuleContext ruleContext, - ImmutableList sources, - ImmutableList sourceJars, - J2ObjcMappingFileProvider depJ2ObjcMappingFileProvider, - NestedSet compileTimeJars, - J2ObjcSource j2ObjcSource) - throws EvalException { - CustomCommandLine.Builder argBuilder = CustomCommandLine.builder(); - ToolchainInfo toolchainInfo = - ruleContext - .getToolchainContexts() - .getToolchainContext("j2objc") - .forToolchainType( - ruleContext - .getPrerequisite(JavaRuleClasses.JAVA_TOOLCHAIN_TYPE_ATTRIBUTE_NAME) - .getLabel()); - JavaToolchainProvider provider = (JavaToolchainProvider) toolchainInfo.getValue("java"); - PathFragment javaExecutable = provider.getJavaRuntime().javaBinaryExecPathFragment(); - argBuilder.add("--java", javaExecutable.getPathString()); - - Artifact j2ObjcDeployJar = ruleContext.getPrerequisiteArtifact("$j2objc"); - argBuilder.addExecPath("--j2objc", j2ObjcDeployJar); - - argBuilder.add("--main_class").add("com.google.devtools.j2objc.J2ObjC"); - argBuilder.add("--objc_file_path").addPath(j2ObjcSource.getObjcFilePath()); - - Artifact outputDependencyMappingFile = j2ObjcOutputDependencyMappingFile(ruleContext); - argBuilder.addExecPath("--output_dependency_mapping_file", outputDependencyMappingFile); - - if (!sourceJars.isEmpty()) { - argBuilder.addExecPaths( - "--src_jars", VectorArg.join(",").each(ImmutableList.copyOf(sourceJars))); - argBuilder.addAll(sourceJarFlags(ruleContext)); - } - - List translationFlags = - ruleContext.getFragment(J2ObjcConfiguration.class).getTranslationFlags(); - argBuilder.addAll(ImmutableList.copyOf(translationFlags)); - - NestedSet depsHeaderMappingFiles = - depJ2ObjcMappingFileProvider.getHeaderMappingFiles(); - if (!depsHeaderMappingFiles.isEmpty()) { - argBuilder.addExecPaths("--header-mapping", VectorArg.join(",").each(depsHeaderMappingFiles)); - } - - boolean experimentalJ2ObjcHeaderMap = - ruleContext.getFragment(J2ObjcConfiguration.class).experimentalJ2ObjcHeaderMap(); - Artifact outputHeaderMappingFile = j2ObjcOutputHeaderMappingFile(ruleContext); - if (!experimentalJ2ObjcHeaderMap) { - argBuilder.addExecPath("--output-header-mapping", outputHeaderMappingFile); - } - - NestedSet depsClassMappingFiles = depJ2ObjcMappingFileProvider.getClassMappingFiles(); - if (!depsClassMappingFiles.isEmpty()) { - argBuilder.addExecPaths("--mapping", VectorArg.join(",").each(depsClassMappingFiles)); - } - - Artifact archiveSourceMappingFile = j2ObjcOutputArchiveSourceMappingFile(ruleContext); - argBuilder.addExecPath("--output_archive_source_mapping_file", archiveSourceMappingFile); - - Artifact compiledLibrary = j2objcIntermediateArtifacts(ruleContext).archive(); - argBuilder.addExecPath("--compiled_archive_file_path", compiledLibrary); - - Artifact bootclasspathJar = ruleContext.getPrerequisiteArtifact("$jre_emul_jar"); - argBuilder.addFormatted("-Xbootclasspath:%s", bootclasspathJar); - - // A valid Java system module contains 3 files. The top directory contains a file "release". - ImmutableList moduleFiles = - ruleContext.getPrerequisiteArtifacts("$jre_emul_module").list(); - for (Artifact a : moduleFiles) { - if (a.getFilename().equals("release")) { - argBuilder.add("--system", a.getDirname()); - break; - } - } - - Artifact deadCodeReport = ruleContext.getPrerequisiteArtifact(":dead_code_report"); - if (deadCodeReport != null) { - argBuilder.addExecPath("--dead-code-report", deadCodeReport); - } - - argBuilder.add("-d").addPath(j2ObjcSource.getObjcFilePath()); - - if (!compileTimeJars.isEmpty()) { - argBuilder.addExecPaths("-classpath", VectorArg.join(":").each(compileTimeJars)); - } - - argBuilder.addExecPaths(sources); - - SpawnAction.Builder transpilationAction = - new SpawnAction.Builder() - .setMnemonic("TranspilingJ2objc") - .setExecutable(ruleContext.getExecutablePrerequisite("$j2objc_wrapper")) - .addInput(j2ObjcDeployJar) - .addInput(bootclasspathJar) - .addInputs(moduleFiles) - .addInputs(sources) - .addInputs(sourceJars) - .addTransitiveInputs(compileTimeJars) - .addTransitiveInputs(provider.getJavaRuntime().javaBaseInputs()) - .addTransitiveInputs(depsHeaderMappingFiles) - .addTransitiveInputs(depsClassMappingFiles) - .addCommandLine( - argBuilder.build(), - ParamFileInfo.builder(ParameterFile.ParameterFileType.UNQUOTED) - .setCharset(ISO_8859_1) - .setUseAlways(true) - .build()) - .addOutputs(j2ObjcSource.getObjcSrcs()) - .addOutputs(j2ObjcSource.getObjcHdrs()) - .addOutput(outputDependencyMappingFile) - .addOutput(archiveSourceMappingFile) - .setExecGroup("j2objc"); - - if (deadCodeReport != null) { - transpilationAction.addInput(deadCodeReport); - } - - if (!experimentalJ2ObjcHeaderMap) { - transpilationAction.addOutput(outputHeaderMappingFile); - } - ruleContext.registerAction(transpilationAction.build(ruleContext)); - - if (experimentalJ2ObjcHeaderMap) { - CustomCommandLine.Builder headerMapCommandLine = CustomCommandLine.builder(); - if (!sources.isEmpty()) { - headerMapCommandLine.addExecPaths("--source_files", VectorArg.join(",").each(sources)); - } - if (!sourceJars.isEmpty()) { - headerMapCommandLine.addExecPaths("--source_jars", VectorArg.join(",").each(sourceJars)); - } - headerMapCommandLine.addExecPath("--output_mapping_file", outputHeaderMappingFile); - ruleContext.registerAction( - new SpawnAction.Builder() - .setMnemonic("GenerateJ2objcHeaderMap") - .setExecutable(ruleContext.getExecutablePrerequisite("$j2objc_header_map")) - .addInputs(sources) - .addInputs(sourceJars) - .addCommandLine( - headerMapCommandLine.build(), - ParamFileInfo.builder(ParameterFileType.SHELL_QUOTED).build()) - .addOutput(outputHeaderMappingFile) - .setExecGroup("j2objc") - .build(ruleContext)); - } - - return new J2ObjcMappingFileProvider( - NestedSetBuilder.stableOrder().add(outputHeaderMappingFile).build(), - NestedSetBuilder.stableOrder().build(), - NestedSetBuilder.stableOrder().add(outputDependencyMappingFile).build(), - NestedSetBuilder.stableOrder().add(archiveSourceMappingFile).build()); - } - - private J2ObjcMappingFileProvider createJ2ObjcProtoCompileActions( - ConfiguredTarget base, - StarlarkInfo protoToolchain, - RuleContext ruleContext, - ImmutableList filteredProtoSources, - J2ObjcSource j2ObjcSource) - throws RuleErrorException, InterruptedException { - ImmutableList outputHeaderMappingFiles = - filteredProtoSources.isEmpty() - ? ImmutableList.of() - : ProtoCommon.declareGeneratedFiles(ruleContext, base, ".j2objc.mapping"); - ImmutableList outputClassMappingFiles = - filteredProtoSources.isEmpty() - ? ImmutableList.of() - : ProtoCommon.declareGeneratedFiles(ruleContext, base, ".clsmap.properties"); - ImmutableList outputs = - ImmutableList.builder() - .addAll(j2ObjcSource.getObjcSrcs()) - .addAll(j2ObjcSource.getObjcHdrs()) - .addAll(outputHeaderMappingFiles) - .addAll(outputClassMappingFiles) - .build(); - - String bindirPath = getProtoOutputRoot(ruleContext).getPathString(); - - ProtoCommon.compile( - ruleContext, - base, - checkNotNull(protoToolchain), - outputs, - bindirPath, - "Generating j2objc proto_library %{label}", - "proto_compiler"); - - return new J2ObjcMappingFileProvider( - NestedSetBuilder.stableOrder().addAll(outputHeaderMappingFiles).build(), - NestedSetBuilder.stableOrder().addAll(outputClassMappingFiles).build(), - NestedSetBuilder.stableOrder().build(), - NestedSetBuilder.stableOrder().build()); - } - - private static List getJ2ObjCMappings(RuleContext context) { - ImmutableList.Builder mappingFileProviderBuilder = - new ImmutableList.Builder<>(); - addJ2ObjCMappingsForAttribute(mappingFileProviderBuilder, context, "deps"); - addJ2ObjCMappingsForAttribute(mappingFileProviderBuilder, context, "runtime_deps"); - addJ2ObjCMappingsForAttribute(mappingFileProviderBuilder, context, "exports"); - return mappingFileProviderBuilder.build(); - } - - private static void addJ2ObjCMappingsForAttribute( - ImmutableList.Builder builder, - RuleContext context, - String attributeName) { - if (context.attributes().has(attributeName, BuildType.LABEL_LIST)) { - for (TransitiveInfoCollection dependencyInfoDatum : context.getPrerequisites(attributeName)) { - J2ObjcMappingFileProvider provider = - dependencyInfoDatum.get(J2ObjcMappingFileProvider.PROVIDER); - if (provider != null) { - builder.add(provider); - } - } - } - } - - private static Artifact j2ObjcOutputHeaderMappingFile(RuleContext ruleContext) { - return ObjcRuleClasses.artifactByAppendingToBaseName(ruleContext, ".mapping.j2objc"); - } - - private static Artifact j2ObjcOutputDependencyMappingFile(RuleContext ruleContext) { - return ObjcRuleClasses.artifactByAppendingToBaseName(ruleContext, ".dependency_mapping.j2objc"); - } - - private static Artifact j2ObjcOutputArchiveSourceMappingFile(RuleContext ruleContext) { - return ObjcRuleClasses.artifactByAppendingToBaseName( - ruleContext, ".archive_source_mapping.j2objc"); - } - - private static Artifact j2ObjcSourceJarTranslatedSourceTreeArtifact(RuleContext ruleContext) { - PathFragment rootRelativePath = - ruleContext.getUniqueDirectory("_j2objc/src_jar_files").getRelative("source_files"); - return ruleContext.getTreeArtifact(rootRelativePath, ruleContext.getBinOrGenfilesDirectory()); - } - - /** - * Returns a unique path fragment for j2objc headers. The slightly shorter path is useful for very - * large app builds, which otherwise may have command lines that are too long to be executable. - */ - private static String j2objcHeaderBase(RuleContext ruleContext) { - boolean shorterPath = - ruleContext.getFragment(J2ObjcConfiguration.class).experimentalShorterHeaderPath(); - return shorterPath ? "_ios" : "_j2objc"; - } - - private static Artifact j2objcSourceJarTranslatedHeaderTreeArtifact(RuleContext ruleContext) { - String uniqueDirectoryPath = j2objcHeaderBase(ruleContext) + "/src_jar_files"; - PathFragment rootRelativePath = - ruleContext.getUniqueDirectory(uniqueDirectoryPath).getRelative("header_files"); - return ruleContext.getTreeArtifact(rootRelativePath, ruleContext.getBinOrGenfilesDirectory()); - } - - private static boolean j2objcCompileWithARC(RuleContext ruleContext) { - return ruleContext.getFragment(J2ObjcConfiguration.class).compileWithARC(); - } - - private static J2ObjcSource javaJ2ObjcSource( - RuleContext ruleContext, - ImmutableList javaInputSourceFiles, - ImmutableList javaSourceJarFiles) { - PathFragment objcFileRootRelativePath = - ruleContext.getUniqueDirectory(j2objcHeaderBase(ruleContext)); - PathFragment objcFileRootExecPath = - ruleContext.getBinFragment().getRelative(objcFileRootRelativePath); - - // Note that these are mutable lists so that we can add the translated file info below. - List objcSrcs = - getOutputObjcFiles(ruleContext, javaInputSourceFiles, objcFileRootRelativePath, ".m"); - List objcHdrs = - getOutputObjcFiles(ruleContext, javaInputSourceFiles, objcFileRootRelativePath, ".h"); - List headerSearchPaths = - j2objcSourceHeaderSearchPaths(ruleContext, objcFileRootExecPath, javaInputSourceFiles); - if (!javaSourceJarFiles.isEmpty()) { - // Add the translated source + header files. - objcSrcs.add(j2ObjcSourceJarTranslatedSourceTreeArtifact(ruleContext)); - Artifact translatedHeader = j2objcSourceJarTranslatedHeaderTreeArtifact(ruleContext); - objcHdrs.add(translatedHeader); - headerSearchPaths.add(translatedHeader.getExecPath()); - } - - return new J2ObjcSource( - ruleContext.getRule().getLabel(), - objcSrcs, - objcHdrs, - objcFileRootExecPath, - SourceType.JAVA, - headerSearchPaths, - j2objcCompileWithARC(ruleContext)); - } - - private static J2ObjcSource protoJ2ObjcSource( - RuleContext ruleContext, ConfiguredTarget protoTarget, ImmutableList protoSources) - throws RuleErrorException, InterruptedException { - PathFragment objcFileRootExecPath = getProtoOutputRoot(ruleContext); - - List headerSearchPaths = - j2objcSourceHeaderSearchPaths(ruleContext, objcFileRootExecPath, protoSources); - - return new J2ObjcSource( - ruleContext.getTarget().getLabel(), - protoSources.isEmpty() - ? ImmutableList.of() - : ProtoCommon.declareGeneratedFiles(ruleContext, protoTarget, ".j2objc.pb.m"), - protoSources.isEmpty() - ? ImmutableList.of() - : ProtoCommon.declareGeneratedFiles(ruleContext, protoTarget, ".j2objc.pb.h"), - objcFileRootExecPath, - SourceType.PROTO, - headerSearchPaths, - /*compileWithARC=*/ false); // generated protos do not support ARC. - } - - private static PathFragment getProtoOutputRoot(RuleContext ruleContext) { - if (ruleContext.getConfiguration().isSiblingRepositoryLayout()) { - return ruleContext.getBinFragment(); - } - return ruleContext - .getBinFragment() - .getRelative(ruleContext.getLabel().getRepository().getExecPath(false)); - } - - private static boolean isProtoRule(ConfiguredTarget base) { - try { - return base.get(ProtoInfo.PROVIDER) != null; - } catch (RuleErrorException e) { - return false; - } - } - - /** Returns a mutable List of objc output files. */ - private static List getOutputObjcFiles( - RuleContext ruleContext, - Collection javaSrcs, - PathFragment objcFileRootRelativePath, - String suffix) { - List objcSources = new ArrayList<>(); - for (Artifact javaSrc : javaSrcs) { - objcSources.add( - ruleContext.getRelatedArtifact( - objcFileRootRelativePath.getRelative(javaSrc.getExecPath()), suffix)); - } - return objcSources; - } - - /** - * Returns a mutable list of header search paths necessary to compile the J2ObjC-generated code - * from a single target. - * - * @param ruleContext the rule context - * @param objcFileRootExecPath the exec path under which all J2ObjC-generated file resides - * @param sourcesToTranslate the source files to be translated by J2ObjC in a single target - */ - private static List j2objcSourceHeaderSearchPaths( - RuleContext ruleContext, - PathFragment objcFileRootExecPath, - Collection sourcesToTranslate) { - PathFragment genRoot = ruleContext.getGenfilesFragment(); - List headerSearchPaths = new ArrayList<>(); - headerSearchPaths.add(objcFileRootExecPath); - // We add another header search path with gen root if we have generated sources to translate. - for (Artifact sourceToTranslate : sourcesToTranslate) { - if (!sourceToTranslate.isSourceArtifact()) { - headerSearchPaths.add(objcFileRootExecPath.getRelative(genRoot)); - return headerSearchPaths; - } - } - - return headerSearchPaths; - } - - /** Sets up and returns an {@link ObjcCommon} object containing the J2ObjC-translated code. */ - private static ObjcCommon common( - ObjcCommon.Purpose purpose, - RuleContext ruleContext, - IntermediateArtifacts intermediateArtifacts, - List transpiledSources, - List transpiledHeaders, - List headerSearchPaths, - List dependentAttributes, - List otherDeps) - throws InterruptedException { - ObjcCommon.Builder builder = new ObjcCommon.Builder(purpose, ruleContext); - - if (!transpiledSources.isEmpty() || !transpiledHeaders.isEmpty()) { - CompilationArtifacts compilationArtifacts; - if (j2objcCompileWithARC(ruleContext)) { - compilationArtifacts = - new CompilationArtifacts( - transpiledSources, - /* nonArcSrcs= */ ImmutableList.of(), - transpiledHeaders, - intermediateArtifacts); - } else { - compilationArtifacts = - new CompilationArtifacts( - /* srcs= */ ImmutableList.of(), - transpiledSources, - transpiledHeaders, - intermediateArtifacts); - } - builder.setCompilationArtifacts(compilationArtifacts); - builder.setHasModuleMap(); - } - - ImmutableList.Builder ccInfos = new ImmutableList.Builder<>(); - for (String attrName : dependentAttributes) { - if (ruleContext.attributes().has(attrName, BuildType.LABEL_LIST) - || ruleContext.attributes().has(attrName, BuildType.LABEL)) { - for (TransitiveInfoCollection dep : ruleContext.getPrerequisites(attrName)) { - CcInfo ccInfo = dep.get(CcInfo.PROVIDER); - if (ccInfo != null) { - ccInfos.add(ccInfo); - } - } - builder.addObjcProviders( - ruleContext.getPrerequisites(attrName, ObjcProvider.STARLARK_CONSTRUCTOR)); - } - } - builder.addCcInfos(ccInfos.build()); - - // We can't just use addDeps since that now takes ConfiguredTargetAndData and we only have - // TransitiveInfoCollections - builder.addObjcProviders( - otherDeps.stream() - .map(d -> d.get(ObjcProvider.STARLARK_CONSTRUCTOR)) - .collect(toImmutableList())); - builder.addCcInfos( - otherDeps.stream().map(d -> d.get(CcInfo.PROVIDER)).collect(toImmutableList())); - - return builder - .addIncludes(headerSearchPaths) - .setIntermediateArtifacts(intermediateArtifacts) - .build(); - } -} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcEntryClassProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcEntryClassProvider.java deleted file mode 100644 index 632d6ec55bf6d5..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcEntryClassProvider.java +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2014 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.rules.objc; - -import com.google.devtools.build.lib.analysis.RuleContext; -import com.google.devtools.build.lib.collect.nestedset.NestedSet; -import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; -import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; -import com.google.devtools.build.lib.packages.BuildType; -import com.google.devtools.build.lib.packages.BuiltinProvider; -import com.google.devtools.build.lib.packages.NativeInfo; -import com.google.errorprone.annotations.CanIgnoreReturnValue; - -/** - * This provider is exported by j2objc_library to export entry class information necessary for - * J2ObjC dead code removal performed at the binary level in ObjC rules. - */ -@Immutable -public final class J2ObjcEntryClassProvider extends NativeInfo { - private final NestedSet entryClasses; - - public static final String NAME = "J2ObjcEntryClassInfo"; - public static final J2ObjcEntryClassProvider.Provider PROVIDER = - new J2ObjcEntryClassProvider.Provider(); - - @Override - public BuiltinProvider getProvider() { - return PROVIDER; - } - - /** - * A builder for J2ObjcEntryClassProvider. - */ - public static class Builder { - private final NestedSetBuilder entryClassesBuilder = NestedSetBuilder.stableOrder(); - - /** - * Constructs a new, empty J2ObjcEntryClassProvider builder. - */ - public Builder() {} - - /** - * Transitively adds the given {@link J2ObjcEntryClassProvider} and all its properties to this - * builder. - * - * @param provider the J2ObjcEntryClassProvider to add - * @return this builder - */ - @CanIgnoreReturnValue - public Builder addTransitive(J2ObjcEntryClassProvider provider) { - entryClassesBuilder.addTransitive(provider.getEntryClasses()); - return this; - } - - /** - * Transitively adds the given {@link J2ObjcEntryClassProvider}s and all their properties to - * this builder. - * - * @param providers the J2ObjcEntryClassProviders to add - * @return this builder - */ - @CanIgnoreReturnValue - public Builder addTransitive(Iterable providers) { - for (J2ObjcEntryClassProvider provider : providers) { - addTransitive(provider); - } - return this; - } - - /** - * Transitively adds all the J2ObjcEntryClassProviders and all their properties that can be - * reached through the "deps" attribute. - * - * @param ruleContext the rule context - * @return this builder - */ - @CanIgnoreReturnValue - public Builder addTransitive(RuleContext ruleContext) { - if (ruleContext.attributes().has("deps", BuildType.LABEL_LIST)) { - addTransitive(ruleContext.getPrerequisites("deps", J2ObjcEntryClassProvider.PROVIDER)); - } - - return this; - } - - /** - * Adds the given entry classes to this builder. See {@link #getEntryClasses()}. - * - * @param entryClasses the entry classes to add - * @return this builder - */ - @CanIgnoreReturnValue - public Builder addEntryClasses(Iterable entryClasses) { - entryClassesBuilder.addAll(entryClasses); - return this; - } - - /** - * Builds a J2ObjcEntryClassProvider from the information in this builder. - * - * @return the J2ObjcEntryClassProvider to be built - */ - public J2ObjcEntryClassProvider build() { - return new J2ObjcEntryClassProvider(entryClassesBuilder.build()); - } - } - - /** - * Constructs a {@link J2ObjcEntryClassProvider} to supply J2ObjC-translated ObjC sources to - * objc_binary for compilation and linking. - * - * @param entryClasses a set of names of Java classes to used as entry point for J2ObjC dead code - * analysis. The Java class names should be in canonical format as defined by the Java - * Language Specification. - */ - private J2ObjcEntryClassProvider(NestedSet entryClasses) { - this.entryClasses = entryClasses; - } - - /** - * Returns a set of entry classes specified on attribute entry_classes of j2objc_library targets - * transitively. - */ - public NestedSet getEntryClasses() { - return entryClasses; - } - - /** Provider */ - public static class Provider extends BuiltinProvider { - public Provider() { - super(J2ObjcEntryClassProvider.NAME, J2ObjcEntryClassProvider.class); - } - } -} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcLibrary.java b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcLibrary.java deleted file mode 100644 index 9bbe0a5ff6a8d3..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcLibrary.java +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2015 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.rules.objc; - -import static com.google.devtools.build.lib.collect.nestedset.Order.STABLE_ORDER; - -import com.google.common.collect.ImmutableList; -import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException; -import com.google.devtools.build.lib.analysis.ConfiguredTarget; -import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; -import com.google.devtools.build.lib.analysis.RuleConfiguredTargetFactory; -import com.google.devtools.build.lib.analysis.RuleContext; -import com.google.devtools.build.lib.analysis.RunfilesProvider; -import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; -import com.google.devtools.build.lib.packages.BuildType; -import com.google.devtools.build.lib.packages.Type; -import com.google.devtools.build.lib.rules.cpp.CcInfo; -import com.google.devtools.build.lib.rules.cpp.CppSemantics; -import java.util.List; -import javax.annotation.Nullable; - -/** - * Implementation for the "j2objc_library" rule, which exports ObjC source files translated from - * Java source files in java_library rules to dependent objc_binary rules for compilation and - * linking into the final application bundle. See {@link J2ObjcLibraryBaseRule} for details. - */ -public class J2ObjcLibrary implements RuleConfiguredTargetFactory { - - protected J2ObjcLibrary(CppSemantics cppSemantics) {} - - public static final String NO_ENTRY_CLASS_ERROR_MSG = - "Entry classes must be specified when flag --compilation_mode=opt is on in order to" - + " perform J2ObjC dead code stripping."; - - public static final ImmutableList J2OBJC_SUPPORTED_RULES = - ImmutableList.of("java_import", "java_library", "java_proto_library", "proto_library"); - - private ObjcCommon common(RuleContext ruleContext) throws InterruptedException { - List depsCcInfos = ruleContext.getPrerequisites("deps", CcInfo.PROVIDER); - return new ObjcCommon.Builder(ObjcCommon.Purpose.LINK_ONLY, ruleContext) - .setCompilationAttributes( - CompilationAttributes.Builder.fromRuleContext(ruleContext).build()) - .addDeps(ruleContext.getPrerequisites("deps")) - .addCcLinkingContexts(depsCcInfos) - .addDeps(ruleContext.getPrerequisites("jre_deps")) - .addDirectCcCompilationContexts(depsCcInfos) - .setIntermediateArtifacts(new IntermediateArtifacts(ruleContext)) - .build(); - } - - private static void j2objcLibraryLockdown(RuleContext ruleContext) throws RuleErrorException { - if (!ruleContext.getFragment(J2ObjcConfiguration.class).j2objcLibraryMigration()) { - return; - } - - if (!ruleContext - .getRule() - .getRuleTags() - .contains("__J2OBJC_LIBRARY_MIGRATION_DO_NOT_USE_WILL_BREAK__")) { - throw ruleContext.throwWithRuleError( - "j2objc_library is locked. Please do not use this rule since it will be deleted in the" - + " future."); - } - } - - @Override - @Nullable - public ConfiguredTarget create(RuleContext ruleContext) - throws InterruptedException, RuleErrorException, ActionConflictException { - j2objcLibraryLockdown(ruleContext); - - checkAttributes(ruleContext); - - if (ruleContext.hasErrors()) { - return null; - } - - J2ObjcEntryClassProvider j2ObjcEntryClassProvider = new J2ObjcEntryClassProvider.Builder() - .addTransitive(ruleContext) - .addEntryClasses(ruleContext.attributes().get("entry_classes", Type.STRING_LIST)) - .build(); - - ObjcCommon common = common(ruleContext); - ObjcProvider objcProvider = common.getObjcProvider(); - - J2ObjcMappingFileProvider j2ObjcMappingFileProvider = - J2ObjcMappingFileProvider.union( - ruleContext.getPrerequisites("deps", J2ObjcMappingFileProvider.PROVIDER)); - - return new RuleConfiguredTargetBuilder(ruleContext) - .setFilesToBuild(NestedSetBuilder.emptySet(STABLE_ORDER)) - .add(RunfilesProvider.class, RunfilesProvider.EMPTY) - .addNativeDeclaredProvider(j2ObjcEntryClassProvider) - .addNativeDeclaredProvider(j2ObjcMappingFileProvider) - .addNativeDeclaredProvider(objcProvider) - .addNativeDeclaredProvider(common.createCcInfo()) - .addStarlarkTransitiveInfo(ObjcProvider.STARLARK_NAME, objcProvider) - .build(); - } - - private static void checkAttributes(RuleContext ruleContext) { - checkAttributes(ruleContext, "deps"); - checkAttributes(ruleContext, "exports"); - } - - private static void checkAttributes(RuleContext ruleContext, String attributeName) { - if (!ruleContext.attributes().has(attributeName, BuildType.LABEL_LIST)) { - return; - } - - List entryClasses = ruleContext.attributes().get("entry_classes", Type.STRING_LIST); - J2ObjcConfiguration j2objcConfiguration = ruleContext.getFragment(J2ObjcConfiguration.class); - if (j2objcConfiguration.removeDeadCode() && (entryClasses == null || entryClasses.isEmpty())) { - ruleContext.attributeError("entry_classes", NO_ENTRY_CLASS_ERROR_MSG); - } - } -} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcLibraryBaseRule.java b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcLibraryBaseRule.java index 87e11a0a7d2d04..90147504a0dc6b 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcLibraryBaseRule.java +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcLibraryBaseRule.java @@ -33,14 +33,10 @@ /** * Abstract rule definition for j2objc_library. + * + *

      This rule is implemented in Starlark. This class remains only for doc-gen purposes. */ public class J2ObjcLibraryBaseRule implements RuleDefinition { - private final J2ObjcAspect j2ObjcAspect; - - public J2ObjcLibraryBaseRule(J2ObjcAspect j2ObjcAspect) { - this.j2ObjcAspect = j2ObjcAspect; - } - @Override public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) { // TODO(rduan): Add support for package prefixes. @@ -81,7 +77,6 @@ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) */ .add( attr("deps", LABEL_LIST) - .aspect(j2ObjcAspect) .allowedRuleClasses( "j2objc_library", "java_library", "java_import", "java_proto_library") .allowedFileTypes()) diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcMappingFileProvider.java b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcMappingFileProvider.java deleted file mode 100644 index 2940f8a9ae3274..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcMappingFileProvider.java +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2014 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.rules.objc; - -import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.collect.nestedset.NestedSet; -import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; -import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; -import com.google.devtools.build.lib.packages.BuiltinProvider; -import com.google.devtools.build.lib.packages.NativeInfo; -import com.google.errorprone.annotations.CanIgnoreReturnValue; - -/** - * This provider is exported by java_library rules and proto_library rules via the j2objc aspect. - */ -@Immutable -public final class J2ObjcMappingFileProvider extends NativeInfo { - - private final NestedSet headerMappingFiles; - private final NestedSet classMappingFiles; - private final NestedSet dependencyMappingFiles; - private final NestedSet archiveSourceMappingFiles; - - public static final String NAME = "J2ObjcMappingFileInfo"; - public static final J2ObjcMappingFileProvider.Provider PROVIDER = - new J2ObjcMappingFileProvider.Provider(); - - @Override - public BuiltinProvider getProvider() { - return PROVIDER; - } - - /** - * Returns a {@link J2ObjcMappingFileProvider} which combines all input - * {@link J2ObjcMappingFileProvider}s. All mapping files present in any of the input providers - * will be present in the output provider. - */ - public static J2ObjcMappingFileProvider union(Iterable providers) { - J2ObjcMappingFileProvider.Builder builder = new J2ObjcMappingFileProvider.Builder(); - for (J2ObjcMappingFileProvider provider : providers) { - builder.addTransitive(provider); - } - - return builder.build(); - } - - /** - * Constructs a {@link J2ObjcMappingFileProvider} with mapping files to export mappings required - * by J2ObjC translation and proto compilation. - * - * @param headerMappingFiles a nested set of header mapping files which map Java classes to - * their associated translated ObjC header. Used by J2ObjC to output correct import directives - * during translation. - * @param classMappingFiles a nested set of class mapping files which map Java class names to - * their associated ObjC class names. Used to support J2ObjC package prefixes. - * @param dependencyMappingFiles a nested set of dependency mapping files which map translated - * ObjC files to their translated direct dependency files. Used to support J2ObjC dead code - * analysis and removal. - * @param archiveSourceMappingFiles a nested set of files containing mappings between J2ObjC - * static library archives and their associated J2ObjC-translated source files. - */ - public J2ObjcMappingFileProvider(NestedSet headerMappingFiles, - NestedSet classMappingFiles, NestedSet dependencyMappingFiles, - NestedSet archiveSourceMappingFiles) { - this.headerMappingFiles = headerMappingFiles; - this.classMappingFiles = classMappingFiles; - this.dependencyMappingFiles = dependencyMappingFiles; - this.archiveSourceMappingFiles = archiveSourceMappingFiles; - } - - /** - * Returns the ObjC header to Java type mapping files for J2ObjC translation. J2ObjC needs these - * mapping files to be able to output translated files with correct header import paths in the - * same directories of the Java source files. - */ - public NestedSet getHeaderMappingFiles() { - return headerMappingFiles; - } - - /** - * Returns the Java class name to ObjC class name mapping files. J2ObjC transpiler and J2ObjC - * proto plugin needs this mapping files to support "objc_class_prefix" proto option, which sets - * the ObjC class prefix on generated protos. - */ - public NestedSet getClassMappingFiles() { - return classMappingFiles; - } - - /** - * Returns the mapping files containing file dependency information among the translated ObjC - * source files. When flag --j2objc_dead_code_removal is specified, they are used to strip unused - * object files inside J2ObjC static libraries before the linking action at binary level. - */ - public NestedSet getDependencyMappingFiles() { - return dependencyMappingFiles; - } - - /** - * Returns the files containing mappings between J2ObjC static library archives and their - * associated J2ObjC-translated source files. When flag --j2objc_dead_code_removal is specified, - * they are used to strip unused object files inside J2ObjC static libraries before the linking - * action at binary level. - */ - public NestedSet getArchiveSourceMappingFiles() { - return archiveSourceMappingFiles; - } - - /** - * A builder for this provider that is optimized for collection information from transitive - * dependencies. - */ - public static final class Builder { - private final NestedSetBuilder headerMappingFiles = NestedSetBuilder.stableOrder(); - private final NestedSetBuilder classMappingFiles = NestedSetBuilder.stableOrder(); - private final NestedSetBuilder depEntryFiles = NestedSetBuilder.stableOrder(); - private final NestedSetBuilder archiveSourceMappingFiles = - NestedSetBuilder.stableOrder(); - - @CanIgnoreReturnValue - public Builder addTransitive(J2ObjcMappingFileProvider provider) { - headerMappingFiles.addTransitive(provider.getHeaderMappingFiles()); - classMappingFiles.addTransitive(provider.getClassMappingFiles()); - depEntryFiles.addTransitive(provider.getDependencyMappingFiles()); - archiveSourceMappingFiles.addTransitive(provider.getArchiveSourceMappingFiles()); - - return this; - } - - public J2ObjcMappingFileProvider build() { - return new J2ObjcMappingFileProvider( - headerMappingFiles.build(), - classMappingFiles.build(), - depEntryFiles.build(), - archiveSourceMappingFiles.build()); - } - } - - /** Provider */ - public static class Provider extends BuiltinProvider { - public Provider() { - super(J2ObjcMappingFileProvider.NAME, J2ObjcMappingFileProvider.class); - } - } -} diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcSource.java b/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcSource.java deleted file mode 100644 index 46465335a47501..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/rules/objc/J2ObjcSource.java +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2014 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.rules.objc; - -import com.google.common.base.Objects; -import com.google.common.collect.ImmutableList; -import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.cmdline.Label; -import com.google.devtools.build.lib.vfs.PathFragment; -import java.util.List; - -/** An object that captures information of ObjC files generated by J2ObjC in a single target. */ -public final class J2ObjcSource { - - /** - * Indicates the type of files from which the ObjC files included in {@link J2ObjcSource} are - * generated. - */ - public enum SourceType { - /** - * Indicates the original file type is java source file. - */ - JAVA, - - /** - * Indicates the original file type is proto file. - */ - PROTO; - } - - private final Label targetLabel; - private final List objcSrcs; - private final List objcHdrs; - private final PathFragment objcFilePath; - private final SourceType sourceType; - private final List headerSearchPaths; - private final boolean compileWithARC; - - /** - * Constructs a J2ObjcSource containing target information for j2objc transpilation. - * - * @param targetLabel the @{code Label} of the associated target. - * @param objcSrcs the {@code Iterable} containing objc source files generated by J2ObjC - * @param objcHdrs the {@code Iterable} containing objc header files generated by J2ObjC - * @param objcFilePath the {@code PathFragment} under which all the generated objc files are. It - * can be used as header search path for objc compilations. - * @param sourceType the type of files from which the ObjC files are generated. - * @param headerSearchPaths the {@code Iterable} of header search paths necessary for compiling - * the generated J2ObjC sources in {@code objcSrcs} - * @param compileWithARC whether the source files were generated to support ARC compilation - */ - public J2ObjcSource( - Label targetLabel, - List objcSrcs, - List objcHdrs, - PathFragment objcFilePath, - SourceType sourceType, - List headerSearchPaths, - boolean compileWithARC) { - this.targetLabel = targetLabel; - this.objcSrcs = ImmutableList.copyOf(objcSrcs); - this.objcHdrs = ImmutableList.copyOf(objcHdrs); - this.objcFilePath = objcFilePath; - this.sourceType = sourceType; - this.headerSearchPaths = ImmutableList.copyOf(headerSearchPaths); - this.compileWithARC = compileWithARC; - } - - /** - * Returns the label of the associated target. - */ - public Label getTargetLabel() { - return targetLabel; - } - - /** Returns the objc source files generated by J2ObjC. */ - public List getObjcSrcs() { - return objcSrcs; - } - - /* - * Returns the objc header files generated by J2ObjC - */ - public List getObjcHdrs() { - return objcHdrs; - } - - /** - * Returns the {@code PathFragment} which represents a directory where the generated ObjC files - * reside. - */ - public PathFragment getObjcFilePath() { - return objcFilePath; - } - - /** Returns a list of header search paths necessary for compiling the generated J2ObjC sources. */ - public List getHeaderSearchPaths() { - return headerSearchPaths; - } - - /** Returns whether output files were generated to support ARC compilation. */ - public boolean compileWithARC() { - return compileWithARC; - } - - @Override - public final boolean equals(Object other) { - if (!(other instanceof J2ObjcSource)) { - return false; - } - - J2ObjcSource that = (J2ObjcSource) other; - return Objects.equal(this.targetLabel, that.targetLabel) - && this.objcSrcs.equals(that.objcSrcs) - && this.objcHdrs.equals(that.objcHdrs) - && Objects.equal(this.objcFilePath, that.objcFilePath) - && this.sourceType == that.sourceType - && this.headerSearchPaths.equals(that.headerSearchPaths) - && this.compileWithARC == that.compileWithARC; - } - - @Override - public int hashCode() { - return Objects.hashCode( - targetLabel, - objcSrcs, - objcHdrs, - objcFilePath, - sourceType, - headerSearchPaths, - compileWithARC); - } -} - diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/MultiArchBinarySupport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/MultiArchBinarySupport.java index 2dd7c5a2333414..824a51c10051bc 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/objc/MultiArchBinarySupport.java +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/MultiArchBinarySupport.java @@ -38,6 +38,9 @@ import com.google.devtools.build.lib.packages.BuiltinProvider; import com.google.devtools.build.lib.packages.Info; import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; +import com.google.devtools.build.lib.packages.StarlarkInfo; +import com.google.devtools.build.lib.packages.StarlarkProvider; +import com.google.devtools.build.lib.packages.StarlarkProviderIdentifier; import com.google.devtools.build.lib.packages.StructImpl; import com.google.devtools.build.lib.rules.apple.AppleCommandLineOptions; import com.google.devtools.build.lib.rules.apple.AppleConfiguration; @@ -55,11 +58,14 @@ import com.google.devtools.build.lib.rules.objc.CompilationSupport.ExtraLinkArgs; import com.google.devtools.build.lib.skyframe.ConfiguredTargetAndData; import com.google.devtools.build.lib.vfs.PathFragment; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import net.starlark.java.eval.Dict; +import net.starlark.java.eval.EvalException; +import net.starlark.java.eval.StarlarkList; /** Support utility for creating multi-arch Apple binaries. */ public class MultiArchBinarySupport { @@ -114,6 +120,28 @@ public MultiArchBinarySupport(RuleContext ruleContext, CppSemantics cppSemantics this.cppSemantics = cppSemantics; } + private StarlarkInfo getStarlarkUnionedJ2objcProvider( + String providerName, + String unionFunctionName, + Iterable infoCollections) + throws RuleErrorException, InterruptedException { + ImmutableList providers = + getTypedProviders( + infoCollections, + StarlarkProviderIdentifier.forKey( + new StarlarkProvider.Key( + Label.parseCanonicalUnchecked("@_builtins//:common/objc/providers.bzl"), + providerName))); + + Object starlarkFunc = ruleContext.getStarlarkDefinedBuiltin(unionFunctionName); + ruleContext.initStarlarkRuleContext(); + return (StarlarkInfo) + ruleContext.callStarlarkOrThrowRuleError( + starlarkFunc, + ImmutableList.of(StarlarkList.immutableCopyOf(providers)), + new HashMap<>()); + } + /** * Registers actions to link a single-platform/architecture Apple binary in a specific * configuration. @@ -141,16 +169,17 @@ public Artifact registerConfigurationSpecificLinkActions( boolean isStampingEnabled, Iterable infoCollections, Map> outputMapCollector) - throws RuleErrorException, InterruptedException { + throws RuleErrorException, InterruptedException, EvalException { IntermediateArtifacts intermediateArtifacts = new IntermediateArtifacts(ruleContext, dependencySpecificConfiguration.config()); - J2ObjcMappingFileProvider j2ObjcMappingFileProvider = - J2ObjcMappingFileProvider.union( - getTypedProviders(infoCollections, J2ObjcMappingFileProvider.PROVIDER)); - J2ObjcEntryClassProvider j2ObjcEntryClassProvider = - new J2ObjcEntryClassProvider.Builder() - .addTransitive(getTypedProviders(infoCollections, J2ObjcEntryClassProvider.PROVIDER)) - .build(); + + StarlarkInfo j2ObjcEntryClassProvider = + getStarlarkUnionedJ2objcProvider( + "J2ObjcEntryClassInfo", "j2objc_entry_class_info_union", infoCollections); + + StarlarkInfo j2ObjcMappingFileProvider = + getStarlarkUnionedJ2objcProvider( + "J2ObjcMappingFileInfo", "j2objc_mapping_file_info_union", infoCollections); CompilationSupport compilationSupport = new CompilationSupport.Builder(ruleContext, cppSemantics) @@ -658,6 +687,15 @@ private static ImmutableList getTypedProviders( .collect(toImmutableList()); } + private static ImmutableList getTypedProviders( + Iterable infoCollections, + StarlarkProviderIdentifier identifier) { + return stream(infoCollections) + .filter(infoCollection -> infoCollection.get(identifier) != null) + .map(infoCollection -> (StarlarkInfo) infoCollection.get(identifier)) + .collect(toImmutableList()); + } + /** Returns providers from a list of {@link ConfiguredTargetAndData} */ public static List getProvidersFromCtads( List ctads) { diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommon.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommon.java index bf1354700798ad..ef3eee2fe32a7b 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommon.java +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcCommon.java @@ -118,6 +118,9 @@ static class Builder { // TODO(b/171413861): remove after objc link info migration. private final List ccLinkingContextsForMerging = new ArrayList<>(); + private static final ImmutableSet J2OBJC_SUPPORTED_RULES = + ImmutableSet.of("java_import", "java_library", "java_proto_library", "proto_library"); + /** * Builder for {@link ObjcCommon} obtaining both attribute data and configuration data from the * given rule context. @@ -373,7 +376,7 @@ ObjcCommon build() { FileType.filter(artifacts.getSrcs(), HEADERS)); if (artifacts.getArchive().isPresent() - && J2ObjcLibrary.J2OBJC_SUPPORTED_RULES.contains(context.getRule().getRuleClass())) { + && J2OBJC_SUPPORTED_RULES.contains(context.getRule().getRuleClass())) { objcProvider.addAll(J2OBJC_LIBRARY, artifacts.getArchive().asSet()); } } diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcStarlarkInternal.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcStarlarkInternal.java index 2aeb5be1bca62f..d40fc4e2a395ce 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcStarlarkInternal.java +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcStarlarkInternal.java @@ -16,7 +16,6 @@ import static com.google.common.collect.ImmutableList.toImmutableList; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; @@ -25,7 +24,6 @@ import com.google.devtools.build.lib.analysis.LocationExpander; import com.google.devtools.build.lib.analysis.TemplateVariableInfo; import com.google.devtools.build.lib.analysis.starlark.StarlarkRuleContext; -import com.google.devtools.build.lib.packages.NativeInfo; import com.google.devtools.build.lib.packages.Type; import com.google.devtools.build.lib.rules.cpp.CcCompilationContext; import com.google.devtools.build.lib.rules.cpp.CcLinkingContext; @@ -238,31 +236,6 @@ public CompilationArtifacts j2objcCreateCompilationArtifacts( intermediateArtifacts); } - @StarlarkMethod( - name = "j2objc_providers_from_deps", - documented = false, - parameters = { - @Param(name = "ctx", positional = false, named = true), - }) - public Sequence createj2objcProvidersFromDeps(StarlarkRuleContext starlarkRuleContext) - throws EvalException { - J2ObjcMappingFileProvider j2ObjcMappingFileProvider = - J2ObjcMappingFileProvider.union( - starlarkRuleContext - .getRuleContext() - .getPrerequisites("deps", J2ObjcMappingFileProvider.PROVIDER)); - J2ObjcEntryClassProvider j2ObjcEntryClassProvider = - new J2ObjcEntryClassProvider.Builder() - .addTransitive( - starlarkRuleContext - .getRuleContext() - .getPrerequisites("deps", J2ObjcEntryClassProvider.PROVIDER)) - .build(); - - return StarlarkList.immutableCopyOf( - ImmutableList.of(j2ObjcEntryClassProvider, j2ObjcMappingFileProvider)); - } - @StarlarkMethod( name = "create_compilation_context", documented = false, diff --git a/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoCommon.java b/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoCommon.java deleted file mode 100644 index 05f88ce7138c55..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoCommon.java +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2015 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.rules.proto; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Interner; -import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.analysis.ConfiguredTarget; -import com.google.devtools.build.lib.analysis.RuleContext; -import com.google.devtools.build.lib.cmdline.BazelModuleContext; -import com.google.devtools.build.lib.cmdline.Label; -import com.google.devtools.build.lib.concurrent.BlazeInterners; -import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; -import com.google.devtools.build.lib.packages.StarlarkInfo; -import com.google.devtools.build.lib.vfs.PathFragment; -import javax.annotation.Nullable; -import net.starlark.java.eval.EvalException; -import net.starlark.java.eval.Module; -import net.starlark.java.eval.Sequence; -import net.starlark.java.eval.Starlark; -import net.starlark.java.eval.StarlarkFunction; -import net.starlark.java.eval.StarlarkList; -import net.starlark.java.eval.StarlarkThread; -import net.starlark.java.eval.Tuple; - -/** Utility functions for proto_library and proto aspect implementations. */ -public class ProtoCommon { - private ProtoCommon() { - throw new UnsupportedOperationException(); - } - - private static final Interner PROTO_SOURCE_ROOT_INTERNER = - BlazeInterners.newWeakInterner(); - - /** - * Returns a memory efficient version of the passed protoSourceRoot. - * - *

      Any sizable proto graph will contain many {@code .proto} sources with the same source root. - * We can't afford to have all of them represented as individual objects in memory. - * - * @param protoSourceRoot - * @return - */ - static PathFragment memoryEfficientProtoSourceRoot(PathFragment protoSourceRoot) { - return PROTO_SOURCE_ROOT_INTERNER.intern(protoSourceRoot); - } - - public static void checkPrivateStarlarkificationAllowlist(StarlarkThread thread) - throws EvalException { - Label label = - ((BazelModuleContext) Module.ofInnermostEnclosingStarlarkFunction(thread).getClientData()) - .label(); - if (!label.getPackageIdentifier().getRepository().toString().equals("@_builtins")) { - throw Starlark.errorf("Rule in '%s' cannot use private API", label.getPackageName()); - } - } - - public static ImmutableList declareGeneratedFiles( - RuleContext ruleContext, ConfiguredTarget protoTarget, String extension) - throws RuleErrorException, InterruptedException { - StarlarkFunction declareGeneratedFiles = - (StarlarkFunction) - ruleContext.getStarlarkDefinedBuiltin("proto_common_declare_generated_files"); - ruleContext.initStarlarkRuleContext(); - Sequence outputs = - (Sequence) - ruleContext.callStarlarkOrThrowRuleError( - declareGeneratedFiles, - ImmutableList.of( - /* actions */ ruleContext.getStarlarkRuleContext().actions(), - /* proto_info */ protoTarget.get(ProtoInfo.PROVIDER.getKey()), - /* extension */ extension), - ImmutableMap.of()); - try { - return Sequence.cast(outputs, Artifact.class, "declare_generated_files").getImmutableList(); - } catch (EvalException e) { - throw new RuleErrorException(e.getMessageWithStack()); - } - } - - public static void compile( - RuleContext ruleContext, - ConfiguredTarget protoTarget, - StarlarkInfo protoLangToolchainInfo, - Iterable generatedFiles, - @Nullable Object pluginOutput, - String progressMessage, - String execGroup) - throws RuleErrorException, InterruptedException { - StarlarkFunction compile = - (StarlarkFunction) ruleContext.getStarlarkDefinedBuiltin("proto_common_compile"); - ruleContext.initStarlarkRuleContext(); - ruleContext.callStarlarkOrThrowRuleError( - compile, - ImmutableList.of( - /* actions */ ruleContext.getStarlarkRuleContext().actions(), - /* proto_info */ protoTarget.get(ProtoInfo.PROVIDER.getKey()), - /* proto_lang_toolchain_info */ protoLangToolchainInfo, - /* generated_files */ StarlarkList.immutableCopyOf(generatedFiles), - /* plugin_output */ pluginOutput == null ? Starlark.NONE : pluginOutput), - ImmutableMap.of( - "experimental_progress_message", - progressMessage, - "experimental_exec_group", - execGroup)); - } - - public static Sequence filterSources( - RuleContext ruleContext, ConfiguredTarget protoTarget, StarlarkInfo protoLangToolchainInfo) - throws RuleErrorException, InterruptedException { - StarlarkFunction filterSources = - (StarlarkFunction) - ruleContext.getStarlarkDefinedBuiltin("proto_common_experimental_filter_sources"); - ruleContext.initStarlarkRuleContext(); - try { - return Sequence.cast( - ((Tuple) - ruleContext.callStarlarkOrThrowRuleError( - filterSources, - ImmutableList.of( - /* proto_info */ protoTarget.get(ProtoInfo.PROVIDER.getKey()), - /* proto_lang_toolchain_info */ protoLangToolchainInfo), - ImmutableMap.of())) - .get(0), - Artifact.class, - "included"); - } catch (EvalException e) { - - throw new RuleErrorException(e.getMessageWithStack()); - } - } -} diff --git a/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoConfiguration.java b/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoConfiguration.java index 99d5b112c04219..8fa47304690328 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoConfiguration.java +++ b/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoConfiguration.java @@ -22,6 +22,7 @@ import com.google.devtools.build.lib.analysis.config.FragmentOptions; import com.google.devtools.build.lib.analysis.config.RequiresOptions; import com.google.devtools.build.lib.analysis.starlark.annotations.StarlarkConfigurationField; +import com.google.devtools.build.lib.cmdline.BazelModuleContext; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; import com.google.devtools.build.lib.starlarkbuildapi.ProtoConfigurationApi; @@ -33,6 +34,8 @@ import java.util.List; import net.starlark.java.annot.StarlarkMethod; import net.starlark.java.eval.EvalException; +import net.starlark.java.eval.Module; +import net.starlark.java.eval.Starlark; import net.starlark.java.eval.StarlarkThread; /** Configuration for Protocol Buffer Libraries. */ @@ -203,13 +206,23 @@ public ImmutableList protocOpts() { return protocOpts; } + private static void checkPrivateStarlarkificationAllowlist(StarlarkThread thread) + throws EvalException { + Label label = + ((BazelModuleContext) Module.ofInnermostEnclosingStarlarkFunction(thread).getClientData()) + .label(); + if (!label.getPackageIdentifier().getRepository().toString().equals("@_builtins")) { + throw Starlark.errorf("Rule in '%s' cannot use private API", label.getPackageName()); + } + } + @StarlarkMethod( name = "experimental_proto_descriptorsets_include_source_info", useStarlarkThread = true, documented = false) public boolean experimentalProtoDescriptorSetsIncludeSourceInfoForStarlark(StarlarkThread thread) throws EvalException { - ProtoCommon.checkPrivateStarlarkificationAllowlist(thread); + checkPrivateStarlarkificationAllowlist(thread); return experimentalProtoDescriptorSetsIncludeSourceInfo(); } @@ -268,13 +281,13 @@ public Label protoToolchainForCc() { @StarlarkMethod(name = "strict_proto_deps", useStarlarkThread = true, documented = false) public String strictProtoDepsForStarlark(StarlarkThread thread) throws EvalException { - ProtoCommon.checkPrivateStarlarkificationAllowlist(thread); + checkPrivateStarlarkificationAllowlist(thread); return strictProtoDeps().toString(); } @StarlarkMethod(name = "strict_public_imports", useStarlarkThread = true, documented = false) public String strictPublicImportsForStarlark(StarlarkThread thread) throws EvalException { - ProtoCommon.checkPrivateStarlarkificationAllowlist(thread); + checkPrivateStarlarkificationAllowlist(thread); return options.strictPublicImports.toString(); } @@ -288,7 +301,7 @@ public StrictDepsMode strictProtoDeps() { documented = false) public List ccProtoLibraryHeaderSuffixesForStarlark(StarlarkThread thread) throws EvalException { - ProtoCommon.checkPrivateStarlarkificationAllowlist(thread); + checkPrivateStarlarkificationAllowlist(thread); return ccProtoLibraryHeaderSuffixes(); } @@ -302,7 +315,7 @@ public List ccProtoLibraryHeaderSuffixes() { documented = false) public List ccProtoLibrarySourceSuffixesForStarlark(StarlarkThread thread) throws EvalException { - ProtoCommon.checkPrivateStarlarkificationAllowlist(thread); + checkPrivateStarlarkificationAllowlist(thread); return ccProtoLibrarySourceSuffixes(); } diff --git a/src/main/java/com/google/devtools/build/lib/runtime/UiStateTracker.java b/src/main/java/com/google/devtools/build/lib/runtime/UiStateTracker.java index 45e0fe26b574a3..e2d55004a5fdde 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/UiStateTracker.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/UiStateTracker.java @@ -527,7 +527,8 @@ void actionStarted(ActionStartedEvent event) { getActionState(action, actionId, event.getNanoTimeStart()); - if (action.getOwner() != null && action.getOwner().getMnemonic().equals("TestRunner")) { + if (action.getOwner() != null + && action.getOwner().getBuildConfigurationMnemonic().equals("TestRunner")) { Label owner = action.getOwner().getLabel(); if (owner != null) { Set testActionsForOwner = testActions.get(owner); @@ -601,7 +602,8 @@ void actionCompletion(ActionCompletionEvent event) { checkNotNull(activeActions.remove(actionId), "%s not active after %s", actionId, event); - if (action.getOwner() != null && action.getOwner().getMnemonic().equals("TestRunner")) { + if (action.getOwner() != null + && action.getOwner().getBuildConfigurationMnemonic().equals("TestRunner")) { Label owner = action.getOwner().getLabel(); if (owner != null) { Set testActionsForOwner = testActions.get(owner); @@ -748,7 +750,8 @@ private String describeActionProgress(ActionState action, int desiredWidth) { protected String describeAction( ActionState actionState, long nanoTime, int desiredWidth, Set toSkip) { ActionExecutionMetadata action = actionState.action; - if (action.getOwner() != null && action.getOwner().getMnemonic().equals("TestRunner")) { + if (action.getOwner() != null + && action.getOwner().getBuildConfigurationMnemonic().equals("TestRunner")) { Label owner = action.getOwner().getLabel(); if (owner != null) { Set allRelatedActions = testActions.get(owner); diff --git a/src/main/java/com/google/devtools/build/lib/shell/BUILD b/src/main/java/com/google/devtools/build/lib/shell/BUILD index 6ea795813746e9..d48b9ccbae4f61 100644 --- a/src/main/java/com/google/devtools/build/lib/shell/BUILD +++ b/src/main/java/com/google/devtools/build/lib/shell/BUILD @@ -20,6 +20,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/jni", "//src/main/java/com/google/devtools/build/lib/util:describable_execution_unit", "//src/main/java/com/google/devtools/build/lib/util:os", + "//src/main/java/com/google/devtools/build/lib/util:string", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//src/main/java/com/google/devtools/build/lib/windows:processes", @@ -45,6 +46,7 @@ bootstrap_java_library( "//src/main/java/com/google/devtools/build/lib/util:describable_execution_unit", "//src/main/java/com/google/devtools/build/lib/util:filetype", "//src/main/java/com/google/devtools/build/lib/util:os", + "//src/main/java/com/google/devtools/build/lib/util:string", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//src/main/java/com/google/devtools/build/lib/windows:processes", "//third_party:auto_value-jars", diff --git a/src/main/java/com/google/devtools/build/lib/shell/JavaSubprocessFactory.java b/src/main/java/com/google/devtools/build/lib/shell/JavaSubprocessFactory.java index 1a45b8a3a275cf..649647c5f2ee24 100644 --- a/src/main/java/com/google/devtools/build/lib/shell/JavaSubprocessFactory.java +++ b/src/main/java/com/google/devtools/build/lib/shell/JavaSubprocessFactory.java @@ -14,12 +14,17 @@ package com.google.devtools.build.lib.shell; +import static com.google.common.collect.ImmutableList.toImmutableList; + +import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.shell.SubprocessBuilder.StreamAction; +import com.google.devtools.build.lib.util.StringUtil; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.lang.ProcessBuilder.Redirect; +import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -150,7 +155,15 @@ private synchronized Process start(ProcessBuilder builder) throws IOException { @Override public Subprocess create(SubprocessBuilder params) throws IOException { ProcessBuilder builder = new ProcessBuilder(); - builder.command(params.getArgv()); + ImmutableList argv = params.getArgv(); + if (Runtime.version().feature() >= 19 + && Objects.equals(System.getProperty("sun.jnu.encoding"), "UTF-8")) { + // On JDK 19 and newer, java.lang.ProcessImpl#start encodes argv using sun.jnu.encoding, so if + // sun.jnu.encoding is set to UTF-8, our argv needs to be UTF-8. (Note that on some platforms, + // for example on macOS, sun.jnu.encoding is hard-coded in the JVM as UTF-8.) + argv = argv.stream().map(StringUtil::decodeBytestringUtf8).collect(toImmutableList()); + } + builder.command(argv); if (params.getEnv() != null) { builder.environment().clear(); builder.environment().putAll(params.getEnv()); diff --git a/src/main/java/com/google/devtools/build/lib/shell/WindowsSubprocess.java b/src/main/java/com/google/devtools/build/lib/shell/WindowsSubprocess.java index dbe3a23c414553..f3a1ae1c0dd7af 100644 --- a/src/main/java/com/google/devtools/build/lib/shell/WindowsSubprocess.java +++ b/src/main/java/com/google/devtools/build/lib/shell/WindowsSubprocess.java @@ -247,8 +247,13 @@ public void waitFor() throws InterruptedException { @Override public synchronized void close() { if (nativeProcess != WindowsProcesses.INVALID) { - stdoutStream.close(); - stderrStream.close(); + // stdoutStream and stderrStream are null if they are redirected to files. + if (stdoutStream != null) { + stdoutStream.close(); + } + if (stderrStream != null) { + stderrStream.close(); + } long process = nativeProcess; nativeProcess = WindowsProcesses.INVALID; WindowsProcesses.deleteProcess(process); diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BUILD b/src/main/java/com/google/devtools/build/lib/skyframe/BUILD index e7100409b5a26c..0838022f2d0d18 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/BUILD +++ b/src/main/java/com/google/devtools/build/lib/skyframe/BUILD @@ -178,6 +178,7 @@ java_library( ":skyframe_error_processor", ":skyframe_executor_repository_helpers_holder", ":skyframe_incremental_build_monitor", + ":skymeld_inconsistency_receiver", ":starlark_builtins_value", ":state_informing_sky_function_environment", ":target_completion_value", @@ -2973,3 +2974,17 @@ java_library( "//third_party:jsr305", ], ) + +java_library( + name = "skymeld_inconsistency_receiver", + srcs = ["SkymeldInconsistencyReceiver.java"], + deps = [ + ":node_dropping_inconsistency_receiver", + ":sky_functions", + "//src/main/java/com/google/devtools/build/skyframe", + "//src/main/java/com/google/devtools/build/skyframe:graph_inconsistency_java_proto", + "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", + "//third_party:guava", + "//third_party:jsr305", + ], +) diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BzlCompileFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/BzlCompileFunction.java index bcfdd33e3a887f..d18bfe2ca3e699 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/BzlCompileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/BzlCompileFunction.java @@ -14,6 +14,7 @@ package com.google.devtools.build.lib.skyframe; +import com.google.common.collect.ImmutableMap; import com.google.common.hash.HashFunction; import com.google.devtools.build.lib.actions.FileValue; import com.google.devtools.build.lib.cmdline.BazelCompileContext; @@ -29,7 +30,6 @@ import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import java.io.IOException; -import java.util.Map; import javax.annotation.Nullable; import net.starlark.java.eval.Module; import net.starlark.java.eval.StarlarkSemantics; @@ -143,7 +143,7 @@ static BzlCompileValue computeInline( return null; } - Map predeclared; + ImmutableMap predeclared; if (key.isSclDialect()) { predeclared = bazelStarlarkEnvironment.getStarlarkGlobals().getSclToplevels(); } else if (key.kind == BzlCompileValue.Kind.BUILTINS) { diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java index 9b9756ce198f20..8fa7842a22039d 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java @@ -228,6 +228,7 @@ public SkyValue compute(SkyKey key, Environment env) throws ReportedException, UnreportedException, DependencyException, InterruptedException { State state = env.getState(() -> new State(storeTransitivePackages)); ConfiguredTargetKey configuredTargetKey = (ConfiguredTargetKey) key.argument(); + Preconditions.checkArgument(!configuredTargetKey.isProxy(), configuredTargetKey); SkyframeBuildView view = buildViewProvider.getSkyframeBuildView(); if (shouldUnblockCpuWorkWhenFetchingDeps) { @@ -248,9 +249,6 @@ public SkyValue compute(SkyKey key, Environment env) } catch (InconsistentNullConfigException e) { // TODO(b/267529852): see if we can remove this. It's not clear the conditions that trigger // InconsistentNullConfigException are even possible. - // - // TODO(b/261521010): propagate this exception once the parent side rule transition is - // deleted. The caller should handle it correctly. return new NonRuleConfiguredTargetValue( new EmptyConfiguredTarget(configuredTargetKey), computeDependenciesState.transitivePackages()); @@ -509,6 +507,10 @@ private void computeTargetAndConfiguration( storedEvents.handle(Event.error(e.getLocation(), e.getMessage())); } throw new ReportedException(e); + case INVALID_VISIBILITY_DEPENDENCY: + // Bubbles the error up to the parent ConfiguredTargetFunction where it will be reported + // with additional context. + throw new DependencyException(error.invalidVisibilityDependency()); case INCONSISTENT_NULL_CONFIG: throw error.inconsistentNullConfig(); } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKey.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKey.java index 513221089fae90..e2282048cbbc6e 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKey.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKey.java @@ -14,11 +14,13 @@ package com.google.devtools.build.lib.skyframe; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.devtools.build.lib.util.HashCodes.hashObjects; import com.google.common.base.MoreObjects; import com.google.devtools.build.lib.actions.ActionLookupKey; +import com.google.devtools.build.lib.actions.ActionLookupKeyOrProxy; import com.google.devtools.build.lib.analysis.ConfiguredTarget; import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; import com.google.devtools.build.lib.cmdline.Label; @@ -41,9 +43,17 @@ * dependency resolution and the rule analysis. * *

      In practice, a ({@link Label} and post-transition {@link BuildConfigurationKey}) pair plus a - * possible execution platform override {@link Label} with special constraints described as follows. + * possible execution platform override {@link Label} with special constraints. To elaborate, in + * order of highest to lowest potential for concern: * - *

      A build should not request keys with equal ({@link Label}, {@link BuildConfigurationValue}) + *

      1. The {@link BuildConfigurationKey} must be post-transition and thus ready for immediate use + * in dependency resolution and analysis. In practice, this means that if the rule has an + * incoming-edge transition (cfg in {@link RuleClass}) or there are global trimming transitions, + * THOSE TRANSITIONS MUST ALREADY BE DONE before creating the key. Failure to do so will lead to + * build graphs with ConfiguredTarget that have seemingly impossible {@link BuildConfigurationValue} + * (due to the skipped transitions). + * + *

      2. A build should not request keys with equal ({@link Label}, {@link BuildConfigurationValue}) * pairs but different execution platform override {@link Label} if the invoked rule will register * actions. (This is potentially OK if all outputs of all registered actions incorporate the * execution platform in their name unless the build also requests keys without an override that @@ -53,40 +63,32 @@ * *

      Note that this key may be used to look up the generating action of an artifact. * + *

      The {@link ConfiguredTargetKey} is not a {@link SkyKey} and must be cast to one using {@link + * ActionLookupKeyOrProxy#toKey}. + * *

      TODO(blaze-configurability-team): Consider just using BuildOptions over a * BuildConfigurationKey. */ -public class ConfiguredTargetKey implements ActionLookupKey { +public abstract class ConfiguredTargetKey implements ActionLookupKeyOrProxy { /** * Cache so that the number of ConfiguredTargetKey instances is {@code O(configured targets)} and * not {@code O(edges between configured targets)}. */ - private static final SkyKey.SkyKeyInterner interner = SkyKey.newInterner(); + private static final SkyKey.SkyKeyInterner interner = SkyKey.newInterner(); - private final Label label; @Nullable private final BuildConfigurationKey configurationKey; private final transient int hashCode; - private ConfiguredTargetKey( - Label label, @Nullable BuildConfigurationKey configurationKey, int hashCode) { - this.label = label; + private ConfiguredTargetKey(@Nullable BuildConfigurationKey configurationKey, int hashCode) { this.configurationKey = configurationKey; this.hashCode = hashCode; } - @Override - public final SkyFunctionName functionName() { - return SkyFunctions.CONFIGURED_TARGET; - } - - @Override - public SkyKeyInterner getSkyKeyInterner() { - return interner; - } - - @Override - public Label getLabel() { - return label; + public Builder toBuilder() { + return builder() + .setConfigurationKey(configurationKey) + .setLabel(getLabel()) + .setExecutionPlatformLabel(getExecutionPlatformLabel()); } @Nullable @@ -95,26 +97,22 @@ public final BuildConfigurationKey getConfigurationKey() { return configurationKey; } - @Override - public final ConfiguredTargetKey toKey() { - return this; - } + public abstract Label getExecutionPlatformLabel(); - @Nullable - public Label getExecutionPlatformLabel() { - return null; + @Override + public final int hashCode() { + return hashCode; } - public final String prettyPrint() { - if (getLabel() == null) { - return "null"; - } - return String.format("%s (%s)", getLabel(), formatConfigurationKey(configurationKey)); + public boolean isProxy() { + return false; } - @Override - public final int hashCode() { - return hashCode; + private static int computeHashCode( + Label label, + @Nullable BuildConfigurationKey configurationKey, + @Nullable Label executionPlatformLabel) { + return hashObjects(label, configurationKey, executionPlatformLabel); } @Override @@ -132,8 +130,19 @@ && getLabel().equals(other.getLabel()) && Objects.equals(getExecutionPlatformLabel(), other.getExecutionPlatformLabel()); } + public String prettyPrint() { + if (getLabel() == null) { + return "null"; + } + return String.format("%s (%s)", getLabel(), formatConfigurationKey(configurationKey)); + } + + private static ConfiguredTargetKey intern(ConfiguredTargetKey key) { + return (ConfiguredTargetKey) interner.intern((SkyKey) key); + } + @Override - public final String toString() { + public String toString() { // TODO(b/162809183): consider reverting to less verbose toString when bug is resolved. MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper(this).add("label", getLabel()).add("config", configurationKey); @@ -143,7 +152,47 @@ public final String toString() { return helper.toString(); } - private static final class ToolchainDependencyConfiguredTargetKey extends ConfiguredTargetKey { + private static final class RealConfiguredTargetKey extends ConfiguredTargetKey + implements ActionLookupKey { + private final Label label; + + private RealConfiguredTargetKey( + Label label, @Nullable BuildConfigurationKey configurationKey, int hashCode) { + super(configurationKey, hashCode); + this.label = label; + } + + static ConfiguredTargetKey create( + Label label, @Nullable BuildConfigurationKey configurationKey) { + int hashCode = computeHashCode(label, configurationKey, /* executionPlatformLabel= */ null); + return intern(new RealConfiguredTargetKey(label, configurationKey, hashCode)); + } + + @Override + public final SkyFunctionName functionName() { + return SkyFunctions.CONFIGURED_TARGET; + } + + @Override + public SkyKeyInterner getSkyKeyInterner() { + return interner; + } + + @Override + public Label getLabel() { + return label; + } + + @Nullable + @Override + public Label getExecutionPlatformLabel() { + return null; + } + } + + private static final class ToolchainDependencyConfiguredTargetKey extends ConfiguredTargetKey + implements ActionLookupKey { + private final Label label; private final Label executionPlatformLabel; private ToolchainDependencyConfiguredTargetKey( @@ -151,21 +200,119 @@ private ToolchainDependencyConfiguredTargetKey( @Nullable BuildConfigurationKey configurationKey, int hashCode, Label executionPlatformLabel) { - super(label, configurationKey, hashCode); + super(configurationKey, hashCode); + this.label = label; this.executionPlatformLabel = checkNotNull(executionPlatformLabel); } + private static ConfiguredTargetKey create( + Label label, + @Nullable BuildConfigurationKey configurationKey, + Label executionPlatformLabel) { + int hashCode = computeHashCode(label, configurationKey, executionPlatformLabel); + return intern( + new ToolchainDependencyConfiguredTargetKey( + label, configurationKey, hashCode, executionPlatformLabel)); + } + + @Override + public SkyFunctionName functionName() { + return SkyFunctions.CONFIGURED_TARGET; + } + + @Override + public Label getLabel() { + return label; + } + @Override public Label getExecutionPlatformLabel() { return executionPlatformLabel; } + + @Override + public SkyKeyInterner getSkyKeyInterner() { + return interner; + } } - public Builder toBuilder() { - return builder() - .setConfigurationKey(configurationKey) - .setLabel(getLabel()) - .setExecutionPlatformLabel(getExecutionPlatformLabel()); + // This class implements SkyKey only so that it can share the interner. It should never be used as + // a SkyKey. + private static final class ProxyConfiguredTargetKey extends ConfiguredTargetKey + implements SkyKey { + private final ConfiguredTargetKey delegate; + + private static ConfiguredTargetKey create( + ConfiguredTargetKey delegate, @Nullable BuildConfigurationKey configurationKey) { + int hashCode = + computeHashCode( + delegate.getLabel(), configurationKey, delegate.getExecutionPlatformLabel()); + return intern(new ProxyConfiguredTargetKey(delegate, configurationKey, hashCode)); + } + + private ProxyConfiguredTargetKey( + ConfiguredTargetKey delegate, + @Nullable BuildConfigurationKey configurationKey, + int hashCode) { + super(configurationKey, hashCode); + checkArgument( + !delegate.isProxy(), "Proxy keys must not be nested: %s %s", delegate, configurationKey); + this.delegate = delegate; + } + + @Override + public SkyFunctionName functionName() { + // ProxyConfiguredTargetKey is never used directly by Skyframe. It must always be cast using + // toKey. + throw new UnsupportedOperationException(); + } + + @Override + public Label getLabel() { + return delegate.getLabel(); + } + + @Override + @Nullable + public Label getExecutionPlatformLabel() { + return delegate.getExecutionPlatformLabel(); + } + + @Override + public ActionLookupKey toKey() { + return (ActionLookupKey) delegate; + } + + @Override + public boolean isProxy() { + return true; + } + + @Override + public Builder toBuilder() { + return new Builder().setDelegate(delegate).setConfigurationKey(getConfigurationKey()); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("delegate", delegate) + .add("config", getConfigurationKey()) + .toString(); + } + + @Override + public String prettyPrint() { + return super.prettyPrint() + + " virtual(" + + formatConfigurationKey(getConfigurationKey()) + + ")"; + } + + @Override + public SkyKeyInterner getSkyKeyInterner() { + return interner; + } } /** Returns a new {@link Builder} to create instances of {@link ConfiguredTargetKey}. */ @@ -188,6 +335,7 @@ public static final class Builder { private Label label = null; private BuildConfigurationKey configurationKey = null; private Label executionPlatformLabel = null; + private ConfiguredTargetKey delegate; private Builder() {} @@ -221,24 +369,33 @@ public Builder setExecutionPlatformLabel(@Nullable Label executionPlatformLabel) return this; } + /** + * If set, creates a {@link ProxyConfiguredTargetKey}. + * + *

      It's invalid to set a label or execution platform label if this is set. Those will be + * defined by the corresponding values of {@code delegate}. + */ + @CanIgnoreReturnValue + public Builder setDelegate(ConfiguredTargetKey delegate) { + this.delegate = delegate; + return this; + } + /** Builds a new {@link ConfiguredTargetKey} based on the supplied data. */ public ConfiguredTargetKey build() { - int hashCode = computeHashCode(label, configurationKey, executionPlatformLabel); - return interner.intern( - executionPlatformLabel == null - ? new ConfiguredTargetKey(label, configurationKey, hashCode) - : new ToolchainDependencyConfiguredTargetKey( - label, configurationKey, hashCode, executionPlatformLabel)); + if (this.delegate != null) { + checkArgument(label == null); + checkArgument(executionPlatformLabel == null); + return ProxyConfiguredTargetKey.create(delegate, configurationKey); + } + if (this.executionPlatformLabel != null) { + return ToolchainDependencyConfiguredTargetKey.create( + label, configurationKey, executionPlatformLabel); + } + return RealConfiguredTargetKey.create(label, configurationKey); } } - private static int computeHashCode( - Label label, - @Nullable BuildConfigurationKey configurationKey, - @Nullable Label executionPlatformLabel) { - return hashObjects(label, configurationKey, executionPlatformLabel); - } - private static String formatConfigurationKey(@Nullable BuildConfigurationKey key) { if (key == null) { return "null"; diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestors.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestors.java index bc0e542cb66a30..14f3571ff33788 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestors.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestors.java @@ -16,6 +16,8 @@ import static com.google.common.collect.ImmutableList.toImmutableList; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import com.google.common.util.concurrent.Futures; @@ -34,8 +36,10 @@ import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.skyframe.Differencer.DiffWithDelta.Delta; import com.google.devtools.build.skyframe.ImmutableDiff; +import com.google.devtools.build.skyframe.InMemoryGraph; +import com.google.devtools.build.skyframe.InMemoryNodeEntry; import com.google.devtools.build.skyframe.SkyKey; -import com.google.devtools.build.skyframe.SkyValue; +import com.google.devtools.build.skyframe.Version; import java.io.IOException; import java.util.Collections; import java.util.HashMap; @@ -62,12 +66,13 @@ * affected ancestor entries of nodes. It is also resilient to diffs which report only a root of * deleted subtree. */ -final class FileSystemValueCheckerInferringAncestors { +public final class FileSystemValueCheckerInferringAncestors { @Nullable private final TimestampGranularityMonitor tsgm; - private final Map graphValues; - private final Map graphDoneValues; + private final InMemoryGraph inMemoryGraph; private final Map nodeStates; private final SyscallCache syscallCache; + private final SkyValueDirtinessChecker skyValueDirtinessChecker; + private final Set valuesToInvalidate = Sets.newConcurrentHashSet(); private final ConcurrentMap valuesToInject = new ConcurrentHashMap<>(); @@ -115,26 +120,39 @@ boolean signalFinishedChild(boolean needsToBeVisited) { private FileSystemValueCheckerInferringAncestors( @Nullable TimestampGranularityMonitor tsgm, - Map graphValues, - Map graphDoneValues, + InMemoryGraph inMemoryGraph, Map nodeStates, - SyscallCache syscallCache) { + SyscallCache syscallCache, + SkyValueDirtinessChecker skyValueDirtinessChecker) { this.tsgm = tsgm; - this.graphValues = graphValues; - this.graphDoneValues = graphDoneValues; this.nodeStates = nodeStates; this.syscallCache = syscallCache; + this.skyValueDirtinessChecker = skyValueDirtinessChecker; + this.inMemoryGraph = inMemoryGraph; } + @VisibleForTesting @SuppressWarnings("ReferenceEquality") - static ImmutableDiff getDiffWithInferredAncestors( + public static ImmutableDiff getDiffWithInferredAncestors( @Nullable TimestampGranularityMonitor tsgm, - Map graphValues, - Map graphDoneValues, + InMemoryGraph inMemoryGraph, Iterable modifiedKeys, int nThreads, - SyscallCache syscallCache) + SyscallCache syscallCache, + SkyValueDirtinessChecker skyValueDirtinessChecker) throws InterruptedException, AbruptExitException { + Map nodeStates = makeNodeVisitStates(modifiedKeys); + return new FileSystemValueCheckerInferringAncestors( + tsgm, + inMemoryGraph, + Collections.unmodifiableMap(nodeStates), + syscallCache, + skyValueDirtinessChecker) + .processEntries(nThreads); + } + + private static Map makeNodeVisitStates( + Iterable modifiedKeys) { Map nodeStates = new HashMap<>(); for (FileStateKey fileStateKey : modifiedKeys) { RootedPath top = fileStateKey.argument(); @@ -161,14 +179,7 @@ static ImmutableDiff getDiffWithInferredAncestors( lastCreated = existingState == null; } } - - return new FileSystemValueCheckerInferringAncestors( - tsgm, - graphValues, - graphDoneValues, - Collections.unmodifiableMap(nodeStates), - syscallCache) - .processEntries(nThreads); + return nodeStates; } private ImmutableDiff processEntries(int nThreads) @@ -253,8 +264,9 @@ private boolean visitEntry( NodeVisitState parentState) throws StatFailedException { FileStateKey key = FileStateValue.key(path); - @Nullable FileStateValue fsv = (FileStateValue) graphValues.get(key); - if (fsv == null) { + @Nullable InMemoryNodeEntry fsvNode = inMemoryGraph.getIfPresent(key); + @Nullable FileStateValue oldFsv = fsvNode != null ? (FileStateValue) fsvNode.toValue() : null; + if (oldFsv == null) { visitUnknownEntry(key, isInferredDirectory, parentState); parentState.addMaybeDeletedChild(path.getRootRelativePath().getBaseName()); return true; @@ -264,33 +276,58 @@ private boolean visitEntry( || (maybeDeletedChildren != null && listingHasEntriesOutsideOf(path, maybeDeletedChildren))) { parentState.markInferredDirectory(); - if (fsv.getType().isDirectory()) { + if (oldFsv.getType().isDirectory()) { return false; } + // TODO(b/287632270) - handle this scenario valuesToInject.put(key, Delta.justNew(FileStateValue.DIRECTORY_FILE_STATE_NODE)); parentListingKey(path).ifPresent(valuesToInvalidate::add); return true; } - FileStateValue newFsv = getNewFileStateValueFromFileSystem(path); - if (!newFsv.equals(fsv)) { - valuesToInject.put(key, Delta.justNew(newFsv)); - } - + @Nullable FileStateValue newFsv = injectAndGetNewFileStateValueIfDirty(path, fsvNode, oldFsv); if (newFsv.getType().exists()) { parentState.markInferredDirectory(); - } else if (fsv.getType().exists()) { + } else if (oldFsv.getType().exists()) { // exists -> not exists -- deletion. parentState.addMaybeDeletedChild(path.getRootRelativePath().getBaseName()); } - boolean typeChanged = newFsv.getType() != fsv.getType(); + boolean typeChanged = newFsv.getType() != oldFsv.getType(); if (typeChanged) { parentListingKey(path).ifPresent(valuesToInvalidate::add); } return typeChanged; } + /** + * Injects the new file state value if dirty. Returns the old file state value if not dirty and + * the new file state value if dirty. + */ + private FileStateValue injectAndGetNewFileStateValueIfDirty( + RootedPath path, InMemoryNodeEntry oldFsvNode, FileStateValue oldFsv) + throws StatFailedException { + Preconditions.checkState(oldFsv != null, "Unexpected null FileStateValue."); + @Nullable Version oldMtsv = oldFsvNode.getMaxTransitiveSourceVersion(); + SkyValueDirtinessChecker.DirtyResult dirtyResult = + skyValueDirtinessChecker.check(oldFsvNode.getKey(), oldFsv, oldMtsv, syscallCache, tsgm); + if (!dirtyResult.isDirty()) { + return oldFsv; + } + @Nullable FileStateValue newFsv = (FileStateValue) dirtyResult.getNewValue(); + if (newFsv == null) { + throw new StatFailedException(path, new IOException("Filesystem access failed.")); + } + @Nullable Version newMtsv = dirtyResult.getNewMaxTransitiveSourceVersion(); + if (newMtsv == null && !skyValueDirtinessChecker.nullMaxTransitiveSourceVersionOk()) { + // TODO(b/287632270) - add test coverage for unexpected null mtsv's + throw new StatFailedException(path, new IOException("Filesystem access failed.")); + } + + valuesToInject.put(oldFsvNode.getKey(), Delta.justNew(newFsv, newMtsv)); + return newFsv; + } + private void visitUnknownEntry( FileStateKey key, boolean isInferredDirectory, NodeVisitState parentState) throws StatFailedException { @@ -298,13 +335,12 @@ private void visitUnknownEntry( // Run stats on unknown files in order to preserve the parent listing if present unless we // already know it has changed. Optional parentListingKey = parentListingKey(path); - @Nullable - DirectoryListingStateValue parentListing = - parentListingKey - // Only look for done listings since already invalidated ones will be reevaluated - // anyway. - .map(k -> (DirectoryListingStateValue) graphDoneValues.get(k)) - .orElse(null); + @Nullable DirectoryListingStateValue parentListing = null; + if (parentListingKey.isPresent()) { + @Nullable InMemoryNodeEntry entry = inMemoryGraph.getIfPresent(parentListingKey.get()); + parentListing = + entry != null && entry.isDone() ? (DirectoryListingStateValue) entry.getValue() : null; + } // No listing/we already know it has changed -- nothing to gain from stats anymore. if (parentListing == null || valuesToInvalidate.contains(parentListingKey.get())) { @@ -319,9 +355,9 @@ private void visitUnknownEntry( // We don't take advantage of isInferredDirectory because we set it only in cases of a present // descendant/done listing which normally cannot exist without having FileStateValue for // ancestors. - FileStateValue value = getNewFileStateValueFromFileSystem(path); - valuesToInject.put(key, Delta.justNew(value)); - if (isInferredDirectory || value.getType().exists()) { + @Nullable FileStateValue newValue = injectAndGetNewFileStateValueForUnknownEntry(path, key); + + if (isInferredDirectory || newValue.getType().exists()) { parentState.markInferredDirectory(); } @@ -329,26 +365,40 @@ private void visitUnknownEntry( Dirent dirent = parentListing.getDirents().maybeGetDirent(path.getRootRelativePath().getBaseName()); @Nullable Dirent.Type typeInListing = dirent != null ? dirent.getType() : null; - if (!Objects.equals(typeInListing, direntTypeFromFileStateType(value.getType()))) { + if (!Objects.equals(typeInListing, direntTypeFromFileStateType(newValue.getType()))) { valuesToInvalidate.add(parentListingKey.get()); } } - private FileStateValue getNewFileStateValueFromFileSystem(RootedPath path) + /** Injects the new file state value for unknown entry. */ + private FileStateValue injectAndGetNewFileStateValueForUnknownEntry(RootedPath path, SkyKey key) throws StatFailedException { - try { - return FileStateValue.create(path, syscallCache, tsgm); - } catch (IOException e) { - throw new StatFailedException(path, e); + @Nullable + FileStateValue newValue = + (FileStateValue) skyValueDirtinessChecker.createNewValue(path, syscallCache, tsgm); + if (newValue == null) { + throw new StatFailedException(path, new IOException("Filesystem access failed.")); + } + Version newMtsv = + skyValueDirtinessChecker.getMaxTransitiveSourceVersionForNewValue(key, newValue); + if (newMtsv == null && !skyValueDirtinessChecker.nullMaxTransitiveSourceVersionOk()) { + // TODO(b/287632270) - add test coverage for unexpected null mtsv's + throw new StatFailedException(path, new IOException("Filesystem access failed.")); } + valuesToInject.put(key, Delta.justNew(newValue, newMtsv)); + return newValue; } private boolean listingHasEntriesOutsideOf(RootedPath path, Set allAffectedEntries) { // TODO(192010830): Try looking up BUILD files if there is no listing -- this is a lookup we // can speculatively try since those files are often checked against. @Nullable + InMemoryNodeEntry nodeEntry = inMemoryGraph.getIfPresent(DirectoryListingStateValue.key(path)); + @Nullable DirectoryListingStateValue listing = - (DirectoryListingStateValue) graphDoneValues.get(DirectoryListingStateValue.key(path)); + nodeEntry != null && nodeEntry.isDone() + ? (DirectoryListingStateValue) nodeEntry.getValue() + : null; if (listing == null) { return false; } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/NodeDroppingInconsistencyReceiver.java b/src/main/java/com/google/devtools/build/lib/skyframe/NodeDroppingInconsistencyReceiver.java index ae2a6ef669483c..5283fa8c43911b 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/NodeDroppingInconsistencyReceiver.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/NodeDroppingInconsistencyReceiver.java @@ -61,7 +61,15 @@ public void noteInconsistencyAndMaybeThrow( */ public static boolean isExpectedInconsistency( SkyKey key, @Nullable Collection otherKeys, Inconsistency inconsistency) { - SkyFunctionName expectedMissingChildType = EXPECTED_MISSING_CHILDREN.get(key.functionName()); + return isExpectedInconsistency(key, otherKeys, inconsistency, EXPECTED_MISSING_CHILDREN); + } + + static boolean isExpectedInconsistency( + SkyKey key, + @Nullable Collection otherKeys, + Inconsistency inconsistency, + ImmutableMap expectedMissingChildrenTypes) { + SkyFunctionName expectedMissingChildType = expectedMissingChildrenTypes.get(key.functionName()); if (expectedMissingChildType == null) { return false; } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java index 10b313f8df89da..bdf21b7fa4ba42 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java @@ -29,6 +29,7 @@ import com.google.devtools.build.lib.actions.CommandLineExpansionException; import com.google.devtools.build.lib.actions.FileValue; import com.google.devtools.build.lib.actions.RemoteArtifactChecker; +import com.google.devtools.build.lib.analysis.AnalysisOptions; import com.google.devtools.build.lib.analysis.AspectValue; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.ConfiguredTarget; @@ -236,14 +237,21 @@ public WorkspaceInfoFromDiff sync( throws InterruptedException, AbruptExitException { if (evaluatorNeedsReset) { if (rewindingPermitted(options)) { + // Currently incompatible with Skymeld i.e. this code path won't be run in Skymeld mode. We + // may need to combine these GraphInconsistencyReceiver implementations in the future. var rewindableReceiver = new RewindableGraphInconsistencyReceiver(); rewindableReceiver.setHeuristicallyDropNodes(heuristicallyDropNodes); - this.inconsistencyReceiver = rewindableReceiver; + inconsistencyReceiver = rewindableReceiver; + } else if (isMergedSkyframeAnalysisExecution() + && ((options.getOptions(AnalysisOptions.class) != null + && options.getOptions(AnalysisOptions.class).discardAnalysisCache) + || !tracksStateForIncrementality() + || heuristicallyDropNodes)) { + inconsistencyReceiver = new SkymeldInconsistencyReceiver(heuristicallyDropNodes); + } else if (heuristicallyDropNodes) { + inconsistencyReceiver = new NodeDroppingInconsistencyReceiver(); } else { - inconsistencyReceiver = - heuristicallyDropNodes - ? new NodeDroppingInconsistencyReceiver() - : GraphInconsistencyReceiver.THROWING; + inconsistencyReceiver = GraphInconsistencyReceiver.THROWING; } // Recreate MemoizingEvaluator so that graph is recreated with correct edge-clearing status, diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyValueDirtinessChecker.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyValueDirtinessChecker.java index 2bea163a8844c4..5451bce1de6b6e 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyValueDirtinessChecker.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyValueDirtinessChecker.java @@ -51,6 +51,15 @@ public Version getMaxTransitiveSourceVersionForNewValue(SkyKey key, SkyValue val return null; } + /** + * Returns whether it is ok for this {@link SkyValueDirtinessChecker} to return a null max + * transitive source version. If this method returns false, a null mtsv would indicate an {@link + * java.io.IOException} was thrown. + */ + public boolean nullMaxTransitiveSourceVersionOk() { + return true; + } + /** * If {@code applies(key)}, returns the result of checking whether this key's value is up to date. */ diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java index 050cec9f14e21b..6c00d2bcc18871 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java @@ -29,6 +29,7 @@ import com.google.common.collect.Sets; import com.google.common.collect.Streams; import com.google.common.eventbus.EventBus; +import com.google.common.flogger.GoogleLogger; import com.google.devtools.build.lib.actions.ActionAnalysisMetadata; import com.google.devtools.build.lib.actions.ActionKeyContext; import com.google.devtools.build.lib.actions.ActionLookupKey; @@ -127,6 +128,8 @@ *

      Covers enough functionality to work as a substitute for {@code BuildView#configureTargets}. */ public final class SkyframeBuildView { + private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + private final ConfiguredTargetFactory factory; private final ArtifactFactory artifactFactory; private final SkyframeExecutor skyframeExecutor; @@ -282,6 +285,7 @@ public void setConfiguration( Event.warn( "--discard_analysis_cache was used in the previous build, " + "discarding analysis cache.")); + logger.atInfo().log("Discarding analysis cache because the previous invocation told us to"); skyframeExecutor.handleAnalysisInvalidatingChange(); } else { String diff = describeConfigurationDifference(configuration, maxDifferencesToShow); @@ -291,6 +295,8 @@ public void setConfiguration( diff + ", discarding analysis cache (this can be expensive, see" + " https://bazel.build/advanced/performance/iteration-speed).")); + logger.atInfo().log( + "Discarding analysis cache because the build configuration changed: %s", diff); // Note that clearing the analysis cache is currently required for correctness. It is also // helpful to save memory. // diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java index 941ca5be46d8a8..2e4ba6b327bf56 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java @@ -88,7 +88,7 @@ import com.google.devtools.build.lib.analysis.ConfiguredTargetValue; import com.google.devtools.build.lib.analysis.Dependency; import com.google.devtools.build.lib.analysis.DependencyKind; -import com.google.devtools.build.lib.analysis.InconsistentNullConfigException; +import com.google.devtools.build.lib.analysis.InvalidVisibilityDependencyException; import com.google.devtools.build.lib.analysis.PlatformOptions; import com.google.devtools.build.lib.analysis.TargetAndConfiguration; import com.google.devtools.build.lib.analysis.TargetConfiguredEvent; @@ -186,6 +186,7 @@ import com.google.devtools.build.lib.skyframe.BuildDriverFunction.TransitiveActionLookupValuesHelper; import com.google.devtools.build.lib.skyframe.DiffAwarenessManager.ProcessableModifiedFileSet; import com.google.devtools.build.lib.skyframe.DirtinessCheckerUtils.ExternalDirtinessChecker; +import com.google.devtools.build.lib.skyframe.DirtinessCheckerUtils.FileDirtinessChecker; import com.google.devtools.build.lib.skyframe.DirtinessCheckerUtils.MissingDiffDirtinessChecker; import com.google.devtools.build.lib.skyframe.DirtinessCheckerUtils.UnionDirtinessChecker; import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction; @@ -1370,15 +1371,19 @@ protected Differencer.Diff getDiff( return FileStateValue.key(RootedPath.toRootedPath(pathEntry, pathFragment)); }); - Map valuesMap = memoizingEvaluator.getValues(); - return FileSystemValueCheckerInferringAncestors.getDiffWithInferredAncestors( tsgm, - valuesMap, - memoizingEvaluator.getDoneValues(), + memoizingEvaluator.getInMemoryGraph(), dirtyFileStateSkyKeys, fsvcThreads, - syscallCache); + syscallCache, + getSkyValueDirtinessCheckerForFiles()); + } + + /** Returns the {@link SkyValueDirtinessChecker} relevant for files. */ + @ForOverride + protected SkyValueDirtinessChecker getSkyValueDirtinessCheckerForFiles() { + return new FileDirtinessChecker(); } /** @@ -1492,6 +1497,11 @@ public void setMergedSkyframeAnalysisExecutionSupplier( this.mergedSkyframeAnalysisExecutionSupplier = mergedSkyframeAnalysisExecutionSupplier; } + boolean isMergedSkyframeAnalysisExecution() { + return mergedSkyframeAnalysisExecutionSupplier != null + && mergedSkyframeAnalysisExecutionSupplier.get(); + } + /** Sets the eventBus to use for posting events. */ public void setEventBus(@Nullable EventBus eventBus) { this.eventBus.set(eventBus); @@ -2577,6 +2587,7 @@ public WorkspaceInfoFromDiff sync( syncPackageLoading(pathPackageLocator, commandId, clientEnv, tsgm, executors, options); if (lastAnalysisDiscarded) { + logger.atInfo().log("Discarding analysis cache because the previous invocation told us to"); dropConfiguredTargetsNow(eventHandler); lastAnalysisDiscarded = false; } @@ -3889,7 +3900,8 @@ public void acceptConfiguredTargetAndData(ConfiguredTargetAndData value, int ind public void acceptConfiguredTargetAndDataError(ConfiguredValueCreationException error) {} @Override - public void acceptConfiguredTargetAndDataError(InconsistentNullConfigException error) {} + public void acceptConfiguredTargetAndDataError( + InvalidVisibilityDependencyException error) {} }; EvaluationResult result; diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkymeldInconsistencyReceiver.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkymeldInconsistencyReceiver.java new file mode 100644 index 00000000000000..4c3c15d50c484a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkymeldInconsistencyReceiver.java @@ -0,0 +1,60 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import static com.google.common.base.Preconditions.checkState; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.skyframe.GraphInconsistencyReceiver; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.proto.GraphInconsistency.Inconsistency; +import java.util.Collection; +import javax.annotation.Nullable; + +/** + * The {@link GraphInconsistencyReceiver} that tolerates inconsistencies resulted in dropping + * pre-execution nodes in Skymeld mode. + */ +public class SkymeldInconsistencyReceiver implements GraphInconsistencyReceiver { + private static final ImmutableMap + SKYMELD_EXPECTED_MISSING_CHILDREN = + ImmutableMap.of(SkyFunctions.ACTION_EXECUTION, SkyFunctions.GLOB); + + private final boolean heuristicallyDropNodes; + + public SkymeldInconsistencyReceiver(boolean heuristicallyDropNodes) { + this.heuristicallyDropNodes = heuristicallyDropNodes; + } + + @Override + public void noteInconsistencyAndMaybeThrow( + SkyKey key, @Nullable Collection otherKeys, Inconsistency inconsistency) { + if (heuristicallyDropNodes + && NodeDroppingInconsistencyReceiver.isExpectedInconsistency( + key, otherKeys, inconsistency)) { + // If `--heuristically_drop_nodes` is enabled, check whether the inconsistency is caused by + // dropped state node. If so, tolerate the inconsistency and return. + return; + } + + checkState( + NodeDroppingInconsistencyReceiver.isExpectedInconsistency( + key, otherKeys, inconsistency, SKYMELD_EXPECTED_MISSING_CHILDREN), + "Unexpected inconsistency: %s, %s, %s", + key, + otherKeys, + inconsistency); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/ActionGraphDump.java b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/ActionGraphDump.java index fa1fbd40b98f42..2dcb8728a33c13 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/ActionGraphDump.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/ActionGraphDump.java @@ -30,8 +30,7 @@ import com.google.devtools.build.lib.analysis.AspectValue; import com.google.devtools.build.lib.analysis.ConfiguredTarget; import com.google.devtools.build.lib.analysis.ConfiguredTargetValue; -import com.google.devtools.build.lib.analysis.SourceManifestAction; -import com.google.devtools.build.lib.analysis.actions.FileWriteAction; +import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction; import com.google.devtools.build.lib.analysis.actions.ParameterFileWriteAction; import com.google.devtools.build.lib.analysis.actions.Substitution; import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction; @@ -198,14 +197,11 @@ private void dumpSingleAction(ConfiguredTarget configuredTarget, ActionAnalysisM actionBuilder.addAllArguments(commandAction.getArguments()); } - if (includeFileWriteContents && action instanceof FileWriteAction) { - FileWriteAction fileWriteAction = (FileWriteAction) action; - actionBuilder.setFileContents(fileWriteAction.getFileContents()); - } - - if (includeFileWriteContents && action instanceof SourceManifestAction) { - SourceManifestAction sourceManifestAction = (SourceManifestAction) action; - actionBuilder.setFileContents(sourceManifestAction.getFileContentsAsString(eventHandler)); + if (includeFileWriteContents + && action instanceof AbstractFileWriteAction.FileContentsProvider) { + String contents = + ((AbstractFileWriteAction.FileContentsProvider) action).getFileContents(eventHandler); + actionBuilder.setFileContents(contents); } // Include the content of param files in output. diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/BUILD b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/BUILD index e60c0caad44d3f..3d396fd61ee7d3 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/BUILD +++ b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/BUILD @@ -18,6 +18,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/actions", "//src/main/java/com/google/devtools/build/lib/actions:artifacts", "//src/main/java/com/google/devtools/build/lib/actions:commandline_item", + "//src/main/java/com/google/devtools/build/lib/analysis:actions/abstract_file_write_action", "//src/main/java/com/google/devtools/build/lib/analysis:actions/parameter_file_write_action", "//src/main/java/com/google/devtools/build/lib/analysis:actions/substitution", "//src/main/java/com/google/devtools/build/lib/analysis:actions/template_expansion_action", diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/StreamedOutputHandler.java b/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/StreamedOutputHandler.java deleted file mode 100644 index 9ca936bad10c06..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/skyframe/actiongraph/v2/StreamedOutputHandler.java +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2019 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.lib.skyframe.actiongraph.v2; - -import static com.google.devtools.build.lib.skyframe.actiongraph.v2.AqueryOutputHandler.OutputType.BINARY; -import static com.google.devtools.build.lib.skyframe.actiongraph.v2.AqueryOutputHandler.OutputType.DELIMITED_BINARY; -import static com.google.devtools.build.lib.skyframe.actiongraph.v2.AqueryOutputHandler.OutputType.TEXT; - -import com.google.common.base.Preconditions; -import com.google.devtools.build.lib.analysis.AnalysisProtosV2.Action; -import com.google.devtools.build.lib.analysis.AnalysisProtosV2.ActionGraphContainer; -import com.google.devtools.build.lib.analysis.AnalysisProtosV2.Artifact; -import com.google.devtools.build.lib.analysis.AnalysisProtosV2.AspectDescriptor; -import com.google.devtools.build.lib.analysis.AnalysisProtosV2.Configuration; -import com.google.devtools.build.lib.analysis.AnalysisProtosV2.DepSetOfFiles; -import com.google.devtools.build.lib.analysis.AnalysisProtosV2.PathFragment; -import com.google.devtools.build.lib.analysis.AnalysisProtosV2.RuleClass; -import com.google.devtools.build.lib.analysis.AnalysisProtosV2.Target; -import com.google.devtools.build.lib.skyframe.actiongraph.v2.PrintTask.ProtoPrintTask; -import com.google.devtools.build.lib.skyframe.actiongraph.v2.PrintTask.StreamedProtoPrintTask; -import com.google.devtools.build.lib.skyframe.actiongraph.v2.PrintTask.TextProtoPrintTask; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.Message; -import java.io.IOException; -import java.io.OutputStream; -import java.io.PrintStream; - -/** - * Manages the various streamed output channels of aquery. This does not support JSON format. - * TODO(b/274595070) Remove this class after the flag flip. - */ -public class StreamedOutputHandler implements AqueryOutputHandler { - private final OutputType outputType; - private final OutputStream outputStream; - private final CodedOutputStream codedOutputStream; - private final PrintStream printStream; - - public StreamedOutputHandler( - OutputType outputType, - OutputStream outputStream, - CodedOutputStream codedOutputStream, - PrintStream printStream) { - this.outputType = outputType; - Preconditions.checkArgument( - outputType == BINARY || outputType == DELIMITED_BINARY || outputType == TEXT, - "Only proto, streamed_proto, textproto outputs should be streamed."); - this.outputStream = outputStream; - this.codedOutputStream = codedOutputStream; - this.printStream = printStream; - } - - @Override - public void outputArtifact(Artifact message) throws IOException { - printMessage(message, ActionGraphContainer.ARTIFACTS_FIELD_NUMBER, "artifacts"); - } - - @Override - public void outputAction(Action message) throws IOException { - printMessage(message, ActionGraphContainer.ACTIONS_FIELD_NUMBER, "actions"); - } - - @Override - public void outputTarget(Target message) throws IOException { - printMessage(message, ActionGraphContainer.TARGETS_FIELD_NUMBER, "targets"); - } - - @Override - public void outputDepSetOfFiles(DepSetOfFiles message) throws IOException { - printMessage(message, ActionGraphContainer.DEP_SET_OF_FILES_FIELD_NUMBER, "dep_set_of_files"); - } - - @Override - public void outputConfiguration(Configuration message) throws IOException { - printMessage(message, ActionGraphContainer.CONFIGURATION_FIELD_NUMBER, "configuration"); - } - - @Override - public void outputAspectDescriptor(AspectDescriptor message) throws IOException { - printMessage( - message, ActionGraphContainer.ASPECT_DESCRIPTORS_FIELD_NUMBER, "aspect_descriptors"); - } - - @Override - public void outputRuleClass(RuleClass message) throws IOException { - printMessage(message, ActionGraphContainer.RULE_CLASSES_FIELD_NUMBER, "rule_classes"); - } - - @Override - public void outputPathFragment(PathFragment message) throws IOException { - printMessage(message, ActionGraphContainer.PATH_FRAGMENTS_FIELD_NUMBER, "path_fragments"); - } - - /** - * Prints the Message to the appropriate output channel. - * - * @param message The message to be printed. - */ - private void printMessage(Message message, int fieldNumber, String messageLabel) - throws IOException { - switch (outputType) { - case BINARY: - ProtoPrintTask.print(codedOutputStream, message, fieldNumber); - break; - case DELIMITED_BINARY: - StreamedProtoPrintTask.print(outputStream, message, fieldNumber); - break; - case TEXT: - TextProtoPrintTask.print(printStream, message, messageLabel); - break; - default: - throw new IllegalStateException("Unknown outputType " + outputType.formatName()); - } - } - - @Override - public void close() throws IOException { - outputStream.flush(); - codedOutputStream.flush(); - printStream.flush(); - } -} diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/cpp/CcModuleApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/cpp/CcModuleApi.java index a1699f1b1b03b3..3b593472395cc2 100755 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/cpp/CcModuleApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/cpp/CcModuleApi.java @@ -1688,6 +1688,7 @@ DebugInfoT mergeCcDebugInfoFromStarlark( positional = false, named = true, documented = false), + @Param(name = "lto_obj_root_prefix", positional = false, named = true, documented = false), @Param(name = "bitcode_file", positional = false, named = true, documented = false), @Param( name = "feature_configuration", @@ -1707,6 +1708,7 @@ DebugInfoT mergeCcDebugInfoFromStarlark( LtoBackendArtifactsT createLtoBackendArtifacts( StarlarkRuleContextT starlarkRuleContext, String ltoOutputRootPrefixString, + String ltoObjRootPrefixString, FileT bitcodeFile, FeatureConfigurationT featureConfigurationForStarlark, CcToolchainProviderT ccToolchain, diff --git a/src/main/java/com/google/devtools/build/lib/vfs/DigestHashFunction.java b/src/main/java/com/google/devtools/build/lib/vfs/DigestHashFunction.java index d03e21649ed1a6..e279d7b95b7c39 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/DigestHashFunction.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/DigestHashFunction.java @@ -25,7 +25,6 @@ import com.google.devtools.common.options.OptionsParsingException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.security.Security; import java.util.HashMap; import java.util.Map.Entry; @@ -68,7 +67,6 @@ public int getDigestMaximumLength() { public static final DigestHashFunction SHA1 = register(Hashing.sha1(), "SHA-1", "SHA1"); public static final DigestHashFunction SHA256 = register(Hashing.sha256(), "SHA-256", "SHA256"); - public static final DigestHashFunction BLAKE3 = register(new Blake3HashFunction(), "BLAKE3"); private final HashFunction hashFunction; private final DigestLength digestLength; @@ -106,9 +104,6 @@ public static DigestHashFunction register( */ public static DigestHashFunction register( HashFunction hash, DigestLength digestLength, String hashName, String... altNames) { - if (hashName == "BLAKE3") { - Security.addProvider(new Blake3Provider()); - } try { MessageDigest.getInstance(hashName); } catch (NoSuchAlgorithmException e) { diff --git a/src/main/java/com/google/devtools/build/lib/vfs/bazel/BUILD b/src/main/java/com/google/devtools/build/lib/vfs/bazel/BUILD new file mode 100644 index 00000000000000..1ab77cdf058c0e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/bazel/BUILD @@ -0,0 +1,29 @@ +load("@rules_java//java:defs.bzl", "java_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//src:__subpackages__"], +) + +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//src:__subpackages__"], +) + +java_library( + name = "bazel", + srcs = glob( + [ + "*.java", + ], + ), + deps = [ + "//src/main/java/com/google/devtools/build/lib:runtime", + "//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception", + "//src/main/java/com/google/devtools/build/lib/jni", + "//src/main/java/com/google/devtools/build/lib/vfs", + "//src/main/java/com/google/devtools/common/options", + "//third_party:guava", + ], +) diff --git a/src/main/java/com/google/devtools/build/lib/vfs/bazel/BazelHashFunctions.java b/src/main/java/com/google/devtools/build/lib/vfs/bazel/BazelHashFunctions.java new file mode 100644 index 00000000000000..bcbad2cda8add5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/bazel/BazelHashFunctions.java @@ -0,0 +1,14 @@ +package com.google.devtools.build.lib.vfs.bazel; + +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.vfs.DigestHashFunction; +import java.security.Security; + +public final class BazelHashFunctions extends BlazeModule { + static { + Security.addProvider(new Blake3Provider()); + } + + public static final DigestHashFunction BLAKE3 = + DigestHashFunction.register(new Blake3HashFunction(), "BLAKE3"); +} diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Blake3HashFunction.java b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3HashFunction.java similarity index 97% rename from src/main/java/com/google/devtools/build/lib/vfs/Blake3HashFunction.java rename to src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3HashFunction.java index 39e2674eb0b1c4..28d2b10ccbf438 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/Blake3HashFunction.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3HashFunction.java @@ -1,4 +1,4 @@ -package com.google.devtools.build.lib.vfs; +package com.google.devtools.build.lib.vfs.bazel; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkPositionIndexes; diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Blake3JNI.java b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3JNI.java similarity index 91% rename from src/main/java/com/google/devtools/build/lib/vfs/Blake3JNI.java rename to src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3JNI.java index 3514482b832010..35bf3cc8057d5f 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/Blake3JNI.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3JNI.java @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package com.google.devtools.build.lib.vfs; +package com.google.devtools.build.lib.vfs.bazel; import com.google.devtools.build.lib.jni.JniLoader; @@ -27,6 +27,8 @@ private Blake3JNI() {} public static final native void blake3_hasher_reset(long self); + public static final native void blake3_hasher_close(long self); + public static final native void blake3_hasher_update(long self, byte[] input, int input_len); public static final native void blake3_hasher_finalize_and_close( diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Blake3MessageDigest.java b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3MessageDigest.java similarity index 92% rename from src/main/java/com/google/devtools/build/lib/vfs/Blake3MessageDigest.java rename to src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3MessageDigest.java index 7703b63dbda224..268f1927954235 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/Blake3MessageDigest.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3MessageDigest.java @@ -1,4 +1,4 @@ -package com.google.devtools.build.lib.vfs; +package com.google.devtools.build.lib.vfs.bazel; import static com.google.common.base.Preconditions.checkState; @@ -22,21 +22,14 @@ public final class Blake3MessageDigest extends MessageDigest implements Hasher { // written, a single JNI call will be made that initializes, hashes, and // cleans up the hasher, rather than making separate calls for each operation. public static final int ONESHOT_THRESHOLD = 8 * 1024; - private static ThreadLocal threadLocalBuffer = new ThreadLocal(); - private ByteBuffer buffer = null; + private ByteBuffer buffer = ByteBuffer.allocate(ONESHOT_THRESHOLD); private long hasher = -1; private boolean isDone; public Blake3MessageDigest() { super("BLAKE3"); - isDone = false; - buffer = threadLocalBuffer.get(); - if (buffer == null) { - buffer = ByteBuffer.allocate(ONESHOT_THRESHOLD); - threadLocalBuffer.set(buffer); - } } private void flush() { @@ -94,10 +87,12 @@ public Object clone() throws CloneNotSupportedException { } public void engineReset() { - if (hasher != -1) { - Blake3JNI.blake3_hasher_reset(hasher); - } - buffer.clear(); + if (hasher != -1) { + Blake3JNI.blake3_hasher_close(hasher); + hasher = -1; + } + buffer.clear(); + isDone = false; } public void engineUpdate(ByteBuffer input) { @@ -132,6 +127,11 @@ public int engineDigest(byte[] buf, int off, int len) throws DigestException { return digestBytes.length; } + @Override + protected void finalize() throws Throwable { + engineReset(); + } + /* The following methods implement the {Hasher} interface. */ @CanIgnoreReturnValue diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Blake3Provider.java b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3Provider.java similarity index 58% rename from src/main/java/com/google/devtools/build/lib/vfs/Blake3Provider.java rename to src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3Provider.java index ca8f151249dad7..3ae91ae876ffef 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/Blake3Provider.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3Provider.java @@ -1,10 +1,10 @@ -package com.google.devtools.build.lib.vfs; +package com.google.devtools.build.lib.vfs.bazel; import java.security.Provider; public final class Blake3Provider extends Provider { public Blake3Provider() { super("BLAKE3Provider", "1.0", "A BLAKE3 digest provider"); - put("MessageDigest.BLAKE3", "com.google.devtools.build.lib.vfs.Blake3MessageDigest"); + put("MessageDigest.BLAKE3", Blake3MessageDigest.class.getName()); } } diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerOptions.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerOptions.java index a121d1c7cb611d..dbb96c833c95f9 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/WorkerOptions.java +++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerOptions.java @@ -50,7 +50,7 @@ public static class MultiResourceConverter extends Converter.Contextless convert(String input) throws OptionsParsingException { // TODO(steinman): Make auto value return a reasonable multiplier of host capacity. - if (input == null || "null".equals(input) || "auto".equals(input)) { + if (input == null || input.equals("null") || input.equals("auto")) { return Maps.immutableEntry(null, null); } int pos = input.indexOf('='); @@ -59,7 +59,7 @@ public Map.Entry convert(String input) throws OptionsParsingExc } String name = input.substring(0, pos); String value = input.substring(pos + 1); - if ("auto".equals(value)) { + if (value.equals("auto")) { return Maps.immutableEntry(name, null); } @@ -75,7 +75,7 @@ public String getTypeDescription() { @Option( name = "worker_max_instances", converter = MultiResourceConverter.class, - defaultValue = "auto", + defaultValue = "null", documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY, effectTags = {OptionEffectTag.EXECUTION, OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS}, help = @@ -101,7 +101,7 @@ public String getTypeDescription() { effectTags = {OptionEffectTag.EXECUTION, OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS}, help = "How many WorkRequests a multiplex worker process may receive in parallel if you use the" - + " 'worker' strategy with --experimental_worker_multiplex. May be specified as " + + " 'worker' strategy with --worker_multiplex. May be specified as " + "[name=value] to give a different value per mnemonic. The limit is based on worker " + "keys, which are differentiated based on mnemonic, but also on startup flags and " + "environment, so there can in some cases be more workers per mnemonic than this " diff --git a/src/main/java/com/google/devtools/build/skyframe/BUILD b/src/main/java/com/google/devtools/build/skyframe/BUILD index d9c920b6d48b6a..139223edf23e88 100644 --- a/src/main/java/com/google/devtools/build/skyframe/BUILD +++ b/src/main/java/com/google/devtools/build/skyframe/BUILD @@ -17,6 +17,7 @@ SKYFRAME_OBJECT_SRCS = [ "SkyFunctionName.java", "SkyKey.java", "SkyValue.java", + "UsePooledLabelInterningFlag.java", "Version.java", ] diff --git a/src/main/java/com/google/devtools/build/skyframe/InMemoryGraph.java b/src/main/java/com/google/devtools/build/skyframe/InMemoryGraph.java index 9e22b28e2e0638..e2849dbd991570 100644 --- a/src/main/java/com/google/devtools/build/skyframe/InMemoryGraph.java +++ b/src/main/java/com/google/devtools/build/skyframe/InMemoryGraph.java @@ -96,4 +96,11 @@ default int valuesSize() { * instances back to weak interner and uninstall current pool. */ void cleanupInterningPool(); + + /** + * Returns the {@link InMemoryNodeEntry} for a given {@link SkyKey} if present in the graph. + * Otherwise, returns null. + */ + @Nullable + InMemoryNodeEntry getIfPresent(SkyKey key); } diff --git a/src/main/java/com/google/devtools/build/skyframe/InMemoryGraphImpl.java b/src/main/java/com/google/devtools/build/skyframe/InMemoryGraphImpl.java index 3ef855f18a8ef0..2645ae1fe0822f 100644 --- a/src/main/java/com/google/devtools/build/skyframe/InMemoryGraphImpl.java +++ b/src/main/java/com/google/devtools/build/skyframe/InMemoryGraphImpl.java @@ -80,7 +80,9 @@ private InMemoryGraphImpl(int initialCapacity, boolean usePooledInterning) { this.usePooledInterning = usePooledInterning; if (usePooledInterning) { SkyKeyInterner.setGlobalPool(new SkyKeyPool()); - LabelInterner.setGlobalPool(new LabelPool()); + if (UsePooledLabelInterningFlag.usePooledLabelInterningFlag()) { + LabelInterner.setGlobalPool(new LabelPool()); + } } } @@ -255,7 +257,9 @@ public void cleanupInterningPool() { e -> { weakInternSkyKey(e.getKey()); - if (!e.isDone() || !e.getKey().functionName().equals(SkyFunctions.PACKAGE)) { + if (!UsePooledLabelInterningFlag.usePooledLabelInterningFlag() + || !e.isDone() + || !e.getKey().functionName().equals(SkyFunctions.PACKAGE)) { return; } @@ -267,6 +271,12 @@ public void cleanupInterningPool() { LabelInterner.setGlobalPool(null); } + @Override + @Nullable + public InMemoryNodeEntry getIfPresent(SkyKey key) { + return nodeMap.get(key); + } + static final class EdgelessInMemoryGraphImpl extends InMemoryGraphImpl { public EdgelessInMemoryGraphImpl(boolean usePooledInterning) { diff --git a/src/main/java/com/google/devtools/build/skyframe/ParallelEvaluator.java b/src/main/java/com/google/devtools/build/skyframe/ParallelEvaluator.java index cd6bd0eacdd0aa..4e865c350416f5 100644 --- a/src/main/java/com/google/devtools/build/skyframe/ParallelEvaluator.java +++ b/src/main/java/com/google/devtools/build/skyframe/ParallelEvaluator.java @@ -216,6 +216,9 @@ private EvaluationResult waitForCompletionAndConstructRe "Scheduler exception only thrown for catastrophe in keep_going evaluation: %s", e); catastrophe = true; + // For b/287183296 + logger.atInfo().withCause(e).log( + "Catastrophic exception in --keep_going mode while evaluating SkyKey: %s", errorKey); } } Preconditions.checkState( @@ -549,6 +552,7 @@ private EvaluationResult constructResult( } } if (!cycleRoots.isEmpty()) { + logger.atInfo().log("Detecting cycles with roots: %s", cycleRoots); cycleDetector.checkForCycles(cycleRoots, result, evaluatorContext); } Preconditions.checkState( diff --git a/src/main/java/com/google/devtools/build/skyframe/SimpleCycleDetector.java b/src/main/java/com/google/devtools/build/skyframe/SimpleCycleDetector.java index e75ee88cb2c919..42280fa7057043 100644 --- a/src/main/java/com/google/devtools/build/skyframe/SimpleCycleDetector.java +++ b/src/main/java/com/google/devtools/build/skyframe/SimpleCycleDetector.java @@ -150,7 +150,7 @@ private static ErrorInfo checkForCycles(SkyKey root, ParallelEvaluatorContext ev // Find out which children have errors. Similar logic to that in Evaluate#run(). List errorDeps = getChildrenErrorsForCycle( - key, directDeps.getAllElementsAsIterable(), entry, evaluatorContext); + key, directDeps.getAllElementsAsIterable(), entry, evaluatorContext, removedDeps); checkState( !errorDeps.isEmpty(), "Node %s was not successfully evaluated, but had no child errors. NodeEntry: %s", @@ -366,7 +366,8 @@ private static List getChildrenErrorsForCycle( SkyKey parent, Iterable children, NodeEntry entryForDebugging, - ParallelEvaluatorContext evaluatorContext) + ParallelEvaluatorContext evaluatorContext, + Set removedDepsForDebugging) throws InterruptedException { List allErrors = new ArrayList<>(); boolean foundCycle = false; @@ -388,10 +389,11 @@ private static List getChildrenErrorsForCycle( } checkState( foundCycle, - "Key %s with entry %s had no cycle beneath it: %s", + "Key %s with entry %s had no cycle beneath it: %s; Removed deps: %s", parent, entryForDebugging, - allErrors); + allErrors, + removedDepsForDebugging); return allErrors; } diff --git a/src/main/java/com/google/devtools/build/skyframe/UsePooledLabelInterningFlag.java b/src/main/java/com/google/devtools/build/skyframe/UsePooledLabelInterningFlag.java new file mode 100644 index 00000000000000..124c66c9542a82 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/UsePooledLabelInterningFlag.java @@ -0,0 +1,44 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.devtools.build.lib.util.TestType; +import java.util.Objects; + +/** + * A startup flag to decide whether some {@link com.google.devtools.build.lib.cmdline.Label}s will + * use {@link com.google.devtools.build.lib.cmdline.LabelInterner} backed by {@link + * com.google.devtools.build.lib.concurrent.PooledInterner} or just the regular bazel weak interner. + * + *

      When this flag is true, {@code LabelInterner} will be applied for all {@code Label}s so that + * they are able to switch between interning the instances between the regular bazel weak interner + * and the static global pool. + * + *

      Applying {@code LabelInterner} can reduce memory overhead of having duplicate {@code Label} + * instances in both weak interner and {@link InMemoryGraphImpl}. + */ +// TODO(b/250641010): This flag is temporary to facilitate a controlled rollout. So it should be +// removed after the new pooled interning for `Label` instances is fully released and stable. +public final class UsePooledLabelInterningFlag { + + private static final boolean USE_POOLED_LABEL_INTERNER = + Objects.equals(System.getProperty("BAZEL_USE_POOLED_LABEL_INTERNER"), "1") + || TestType.isInTest(); + + public static boolean usePooledLabelInterningFlag() { + return USE_POOLED_LABEL_INTERNER; + } + + private UsePooledLabelInterningFlag() {} +} diff --git a/src/main/java/com/google/devtools/common/options/processor/OptionProcessor.java b/src/main/java/com/google/devtools/common/options/processor/OptionProcessor.java index fc148014e36b91..8ee41190bd648a 100644 --- a/src/main/java/com/google/devtools/common/options/processor/OptionProcessor.java +++ b/src/main/java/com/google/devtools/common/options/processor/OptionProcessor.java @@ -492,8 +492,7 @@ private static boolean hasSpecialNullDefaultValue(Option annotation) { // Production multiple options that still have default value. // Mostly due to backward compatibility reasons. "runs_per_test", - "flaky_test_attempts", - "worker_max_instances"); + "flaky_test_attempts"); private static boolean isMultipleOptionDefaultValueException(Option annotation) { return MULTIPLE_OPTIONS_DEFAULT_VALUE_EXCEPTIONS.contains(annotation.name()); diff --git a/src/main/java/net/starlark/java/eval/Dict.java b/src/main/java/net/starlark/java/eval/Dict.java index 649ec3520943fb..6ff9a7eb66f474 100644 --- a/src/main/java/net/starlark/java/eval/Dict.java +++ b/src/main/java/net/starlark/java/eval/Dict.java @@ -441,31 +441,35 @@ public static Dict copyOf(@Nullable Mutability mu, Map dict = (Dict) m; // safe - return dict; - } - if (mu == Mutability.IMMUTABLE) { if (m.isEmpty()) { return empty(); } + + if (m instanceof ImmutableMap) { + m.forEach( + (k, v) -> { + Starlark.checkValid(k); + Starlark.checkValid(v); + }); + @SuppressWarnings("unchecked") + var immutableMap = (ImmutableMap) m; + return new Dict<>(immutableMap); + } + + if (m instanceof Dict && ((Dict) m).isImmutable()) { + @SuppressWarnings("unchecked") + var dict = (Dict) m; + return dict; + } + ImmutableMap.Builder immutableMapBuilder = ImmutableMap.builderWithExpectedSize(m.size()); - for (Map.Entry e : m.entrySet()) { - immutableMapBuilder.put( - Starlark.checkValid(e.getKey()), // - Starlark.checkValid(e.getValue())); - } + m.forEach((k, v) -> immutableMapBuilder.put(Starlark.checkValid(k), Starlark.checkValid(v))); return new Dict<>(immutableMapBuilder.buildOrThrow()); } else { LinkedHashMap linkedHashMap = Maps.newLinkedHashMapWithExpectedSize(m.size()); - for (Map.Entry e : m.entrySet()) { - linkedHashMap.put( - Starlark.checkValid(e.getKey()), // - Starlark.checkValid(e.getValue())); - } + m.forEach((k, v) -> linkedHashMap.put(Starlark.checkValid(k), Starlark.checkValid(v))); return new Dict<>(mu, linkedHashMap); } } diff --git a/src/main/java/net/starlark/java/eval/Module.java b/src/main/java/net/starlark/java/eval/Module.java index 3ddf8dbd39a316..740f275247c5bc 100644 --- a/src/main/java/net/starlark/java/eval/Module.java +++ b/src/main/java/net/starlark/java/eval/Module.java @@ -16,6 +16,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import java.util.Arrays; import java.util.HashSet; import java.util.LinkedHashMap; @@ -50,8 +51,9 @@ */ public final class Module implements Resolver.Module { - // The module's predeclared environment. Excludes UNIVERSE bindings. - private ImmutableMap predeclared; + // The module's predeclared environment. Excludes UNIVERSE bindings. Values that are conditionally + // present are stored as GuardedValues regardless of whether they are actually enabled. + private final ImmutableMap predeclared; // The module's global variables, in order of creation. private final LinkedHashMap globalIndex = new LinkedHashMap<>(); @@ -61,12 +63,18 @@ public final class Module implements Resolver.Module { // Its toString appears to Starlark in str(function): "". @Nullable private final Object clientData; + private final StarlarkSemantics semantics; + // An optional doc string for the module. Set after construction when evaluating a .bzl file. @Nullable private String documentation; - private Module(ImmutableMap predeclared, Object clientData) { + private Module( + ImmutableMap predeclared, + @Nullable Object clientData, + StarlarkSemantics semantics) { this.predeclared = predeclared; this.clientData = clientData; + this.semantics = semantics; } /** @@ -85,7 +93,7 @@ public static Module withPredeclared( */ public static Module withPredeclaredAndData( StarlarkSemantics semantics, Map predeclared, @Nullable Object clientData) { - return new Module(filter(predeclared, semantics, clientData), clientData); + return new Module(ImmutableMap.copyOf(predeclared), clientData, semantics); } /** @@ -93,7 +101,8 @@ public static Module withPredeclaredAndData( * Starlark#UNIVERSE}, and with no client data. */ public static Module create() { - return new Module(/*predeclared=*/ ImmutableMap.of(), null); + return new Module( + /* predeclared= */ ImmutableMap.of(), /* clientData= */ null, StarlarkSemantics.DEFAULT); } /** @@ -128,26 +137,18 @@ public static Module ofInnermostEnclosingStarlarkFunction(StarlarkThread thread) } /** - * Returns a map in which each {@link GuardedValue} that is enabled has been replaced by the value - * it guards. Disabled {@code GuardedValues} are left in place for error reporting upon access, - * and should be treated as unavailable. + * Replaces an enabled {@link GuardedValue} with the value it guards. * - *

      The iteration order is unchanged. + *

      A disabled {@link GuardedValue} is left in place for error reporting upon access, and should + * be treated as unavailable. */ - private static ImmutableMap filter( - Map predeclared, StarlarkSemantics semantics, @Nullable Object clientData) { - ImmutableMap.Builder filtered = ImmutableMap.builder(); - for (Map.Entry bind : predeclared.entrySet()) { - Object v = bind.getValue(); - if (v instanceof GuardedValue) { - GuardedValue gv = (GuardedValue) bind.getValue(); - if (gv.isObjectAccessibleUsingSemantics(semantics, clientData)) { - v = gv.getObject(); - } - } - filtered.put(bind.getKey(), v); + private Object filterGuardedValue(Object v) { + Preconditions.checkNotNull(v); + if (!(v instanceof GuardedValue)) { + return v; } - return filtered.build(); + GuardedValue gv = (GuardedValue) v; + return gv.isObjectAccessibleUsingSemantics(semantics, clientData) ? gv.getObject() : gv; } /** Returns the client data associated with this module. */ @@ -175,9 +176,19 @@ public String getDocumentation() { return documentation; } - /** Returns the value of a predeclared (not universal) binding in this module. */ - Object getPredeclared(String name) { - return predeclared.get(name); + /** + * Returns the value of a predeclared (not universal) binding in this module. + * + *

      In the case that the predeclared is a {@link GuardedValue}: If it is enabled, the underlying + * value is returned, otherwise the {@code GuardedValue} itself is returned for error reporting. + */ + @Nullable + public Object getPredeclared(String name) { + var value = predeclared.get(name); + if (value == null) { + return null; + } + return filterGuardedValue(value); } /** @@ -186,8 +197,8 @@ Object getPredeclared(String name) { *

      The map reflects any filtering of {@link GuardedValue}: enabled ones are replaced by the * underlying values that they guard, while disabled ones are left in place for error reporting. */ - public ImmutableMap getPredeclaredBindings() { - return predeclared; + public Map getPredeclaredBindings() { + return Maps.transformValues(predeclared, this::filterGuardedValue); } /** @@ -205,7 +216,7 @@ public ImmutableMap getGlobals() { m.put(e.getKey(), v); } } - return m.build(); + return m.buildOrThrow(); } /** Implements the resolver's module interface. */ @@ -217,7 +228,7 @@ public Resolver.Scope resolve(String name) throws Undefined { } // predeclared? - Object v = predeclared.get(name); + Object v = getPredeclared(name); if (v != null) { if (v instanceof GuardedValue) { // Name is correctly spelled, but access is disabled by a flag or by client data. diff --git a/src/main/native/blake3_jni.cc b/src/main/native/blake3_jni.cc index 8522e43ff725e0..555ad53f56050b 100644 --- a/src/main/native/blake3_jni.cc +++ b/src/main/native/blake3_jni.cc @@ -31,23 +31,29 @@ void release_byte_array(JNIEnv *env, jbyteArray array, jbyte *addr) { } extern "C" JNIEXPORT jlong JNICALL -Java_com_google_devtools_build_lib_vfs_Blake3JNI_allocate_1and_1initialize_1hasher( +Java_com_google_devtools_build_lib_vfs_bazel_Blake3JNI_allocate_1and_1initialize_1hasher( JNIEnv *env, jobject obj) { blake3_hasher *hasher = new blake3_hasher; blake3_hasher_init(hasher); return (jlong)hasher; } -extern "C" JNIEXPORT jlong JNICALL +extern "C" JNIEXPORT void JNICALL Java_com_google_devtools_build_lib_hash_Blake3JNI_blake3_1hasher_1reset( JNIEnv *env, jobject obj, jlong self) { blake3_hasher *hasher = hasher_ptr(self); blake3_hasher_reset(hasher); - return (jlong)hasher; } extern "C" JNIEXPORT void JNICALL -Java_com_google_devtools_build_lib_vfs_Blake3JNI_blake3_1hasher_1update( +Java_com_google_devtools_build_lib_hash_Blake3JNI_blake3_1hasher_1close( + JNIEnv *env, jobject obj, jlong self) { + blake3_hasher *hasher = hasher_ptr(self); + delete hasher; +} + +extern "C" JNIEXPORT void JNICALL +Java_com_google_devtools_build_lib_vfs_bazel_Blake3JNI_blake3_1hasher_1update( JNIEnv *env, jobject obj, jlong self, jbyteArray input, jint input_len) { jbyte *input_addr = get_byte_array(env, input); @@ -56,7 +62,7 @@ Java_com_google_devtools_build_lib_vfs_Blake3JNI_blake3_1hasher_1update( } extern "C" JNIEXPORT void JNICALL -Java_com_google_devtools_build_lib_vfs_Blake3JNI_oneshot( +Java_com_google_devtools_build_lib_vfs_bazel_Blake3JNI_oneshot( JNIEnv *env, jobject obj, jbyteArray input, jint input_len, jbyteArray out, jint out_len) { blake3_hasher *hasher = new blake3_hasher; @@ -74,7 +80,7 @@ Java_com_google_devtools_build_lib_vfs_Blake3JNI_oneshot( } extern "C" JNIEXPORT void JNICALL -Java_com_google_devtools_build_lib_vfs_Blake3JNI_blake3_1hasher_1finalize_1and_1close( +Java_com_google_devtools_build_lib_vfs_bazel_Blake3JNI_blake3_1hasher_1finalize_1and_1close( JNIEnv *env, jobject obj, jlong self, jbyteArray out, jint out_len) { blake3_hasher *hasher = hasher_ptr(self); diff --git a/src/main/native/darwin/fsevents.cc b/src/main/native/darwin/fsevents.cc index 7dfdcc0fc5dc46..76e933cbd07978 100644 --- a/src/main/native/darwin/fsevents.cc +++ b/src/main/native/darwin/fsevents.cc @@ -57,7 +57,7 @@ void FsEventsDiffAwarenessCallback(ConstFSEventStreamRef streamRef, JNIEventsDiffAwareness *info = static_cast(clientCallBackInfo); pthread_mutex_lock(&(info->mutex)); - for (int i = 0; i < numEvents; i++) { + for (size_t i = 0; i < numEvents; i++) { if ((eventFlags[i] & kFSEventStreamEventFlagMustScanSubDirs) != 0) { // Either we lost events or they were coalesced. Assume everything changed // and give up, which matches the fsevents documentation in that the diff --git a/src/main/starlark/builtins_bzl/bazel/exports.bzl b/src/main/starlark/builtins_bzl/bazel/exports.bzl index 0b2035db5b6fa3..0ef0d2dbc49825 100644 --- a/src/main/starlark/builtins_bzl/bazel/exports.bzl +++ b/src/main/starlark/builtins_bzl/bazel/exports.bzl @@ -40,8 +40,8 @@ exported_rules = { "+java_import": java_import, "java_proto_library": java_proto_library, "+cc_proto_library": cc_proto_library, - "+java_binary": java_binary, - "+java_test": java_test, + "java_binary": java_binary, + "java_test": java_test, "py_binary": py_binary, "py_test": py_test, "py_library": py_library, diff --git a/src/main/starlark/builtins_bzl/common/cc/cc_binary.bzl b/src/main/starlark/builtins_bzl/common/cc/cc_binary.bzl index b889808d747b19..945a21f53bf548 100644 --- a/src/main/starlark/builtins_bzl/common/cc/cc_binary.bzl +++ b/src/main/starlark/builtins_bzl/common/cc/cc_binary.bzl @@ -15,7 +15,7 @@ """cc_binary Starlark implementation replacing native""" load(":common/cc/semantics.bzl", "semantics") -load(":common/cc/cc_shared_library.bzl", "CcSharedLibraryInfo", "GraphNodeInfo", "build_exports_map_from_only_dynamic_deps", "build_link_once_static_libs_map", "merge_cc_shared_library_infos", "separate_static_and_dynamic_link_libraries", "sort_linker_inputs", "throw_linked_but_not_exported_errors") +load(":common/cc/cc_shared_library.bzl", "GraphNodeInfo", "add_unused_dynamic_deps", "build_exports_map_from_only_dynamic_deps", "build_link_once_static_libs_map", "merge_cc_shared_library_infos", "separate_static_and_dynamic_link_libraries", "sort_linker_inputs", "throw_linked_but_not_exported_errors") load(":common/cc/cc_helper.bzl", "cc_helper", "linker_mode") load(":common/cc/cc_info.bzl", "CcInfo") load(":common/cc/cc_common.bzl", "cc_common") @@ -331,22 +331,6 @@ def _collect_transitive_dwo_artifacts(cc_compilation_outputs, cc_debug_context, transitive_dwo_files = cc_debug_context.files return depset(dwo_files, transitive = [transitive_dwo_files]) -def _build_map_direct_dynamic_dep_to_transitive_dynamic_deps(ctx): - all_dynamic_dep_linker_inputs = {} - direct_dynamic_dep_to_transitive_dynamic_deps = {} - for dep in ctx.attr.dynamic_deps: - owner = dep[CcSharedLibraryInfo].linker_input.owner - all_dynamic_dep_linker_inputs[owner] = dep[CcSharedLibraryInfo].linker_input - transitive_dynamic_dep_labels = [] - for dynamic_dep in dep[CcSharedLibraryInfo].dynamic_deps.to_list(): - all_dynamic_dep_linker_inputs[dynamic_dep[1].owner] = dynamic_dep[1] - transitive_dynamic_dep_labels.append(dynamic_dep[1].owner) - transitive_dynamic_dep_labels_set = depset(transitive_dynamic_dep_labels, order = "topological") - for export in dep[CcSharedLibraryInfo].exports: - direct_dynamic_dep_to_transitive_dynamic_deps[export] = transitive_dynamic_dep_labels_set - - return direct_dynamic_dep_to_transitive_dynamic_deps, all_dynamic_dep_linker_inputs - def _filter_libraries_that_are_linked_dynamically(ctx, feature_configuration, cc_linking_context): merged_cc_shared_library_infos = merge_cc_shared_library_infos(ctx) link_once_static_libs_map = build_link_once_static_libs_map(merged_cc_shared_library_infos) @@ -365,19 +349,15 @@ def _filter_libraries_that_are_linked_dynamically(ctx, feature_configuration, cc # Entries in unused_dynamic_linker_inputs will be marked None if they are # used - ( - transitive_dynamic_dep_labels, - unused_dynamic_linker_inputs, - ) = _build_map_direct_dynamic_dep_to_transitive_dynamic_deps(ctx) - ( targets_to_be_linked_statically_map, targets_to_be_linked_dynamically_set, topologically_sorted_labels, + unused_dynamic_linker_inputs, ) = separate_static_and_dynamic_link_libraries( + ctx, graph_structure_aspect_nodes, can_be_linked_dynamically, - transitive_dynamic_dep_labels, ) topologically_sorted_labels = [ctx.label] + topologically_sorted_labels @@ -409,35 +389,9 @@ def _filter_libraries_that_are_linked_dynamically(ctx, feature_configuration, cc # Unlike Unix on Windows every dynamic dependency must be linked to the # main binary, even indirect ones that are dependencies of direct - # dynamic dependencies of this binary. So even though linking indirect - # dynamic dependencies is not needed for Unix, we link them here for tests too - # because we cannot know whether the shared libraries were linked with - # RUNPATH or RPATH. If they were linked with the former, then the loader - # won't search in the runfiles directory of this binary for the library, it - # will only search in the RUNPATH set at the time of linking the shared - # library and we cannot possibly know at that point the runfiles directory - # of all of its dependents. - link_indirect_deps = ( - ctx.attr._is_test or - cc_common.is_enabled(feature_configuration = feature_configuration, feature_name = "targets_windows") or - cc_common.is_enabled( - feature_configuration = feature_configuration, - feature_name = "link_indirect_dynamic_deps_in_binary", - ) - ) - direct_dynamic_dep_labels = {dep[CcSharedLibraryInfo].linker_input.owner: True for dep in ctx.attr.dynamic_deps} - topologically_sorted_labels_set = {label: True for label in topologically_sorted_labels} - for dynamic_linker_input_owner, unused_linker_input in unused_dynamic_linker_inputs.items(): - should_link_input = (unused_linker_input and - (link_indirect_deps or dynamic_linker_input_owner in direct_dynamic_dep_labels)) - if should_link_input: - _add_linker_input_to_dict( - dynamic_linker_input_owner, - unused_dynamic_linker_inputs[dynamic_linker_input_owner], - ) - linker_inputs_count += 1 - if dynamic_linker_input_owner not in topologically_sorted_labels_set: - topologically_sorted_labels.append(dynamic_linker_input_owner) + # dynamic dependencies of this binary. + link_indirect_deps = cc_common.is_enabled(feature_configuration = feature_configuration, feature_name = "targets_windows") + linker_inputs_count += add_unused_dynamic_deps(ctx, unused_dynamic_linker_inputs, _add_linker_input_to_dict, topologically_sorted_labels, link_indirect_deps) throw_linked_but_not_exported_errors(linked_statically_but_not_exported) diff --git a/src/main/starlark/builtins_bzl/common/cc/cc_common.bzl b/src/main/starlark/builtins_bzl/common/cc/cc_common.bzl index 1d01ce08d1dc91..b2375707d18443 100644 --- a/src/main/starlark/builtins_bzl/common/cc/cc_common.bzl +++ b/src/main/starlark/builtins_bzl/common/cc/cc_common.bzl @@ -741,6 +741,7 @@ def _create_lto_backend_artifacts( *, ctx, lto_output_root_prefix, + lto_obj_root_prefix, bitcode_file, feature_configuration, cc_toolchain, @@ -753,6 +754,7 @@ def _create_lto_backend_artifacts( ctx = ctx, bitcode_file = bitcode_file, lto_output_root_prefix = lto_output_root_prefix, + lto_obj_root_prefix = lto_obj_root_prefix, feature_configuration = feature_configuration, cc_toolchain = cc_toolchain, fdo_context = fdo_context, diff --git a/src/main/starlark/builtins_bzl/common/cc/cc_shared_library.bzl b/src/main/starlark/builtins_bzl/common/cc/cc_shared_library.bzl index e82b34b2cb995b..690e6466d6506f 100644 --- a/src/main/starlark/builtins_bzl/common/cc/cc_shared_library.bzl +++ b/src/main/starlark/builtins_bzl/common/cc/cc_shared_library.bzl @@ -83,9 +83,14 @@ def _sort_linker_inputs(topologically_sorted_labels, label_to_linker_inputs, lin # dynamically. The transitive_dynamic_dep_labels parameter is only needed for # binaries because they link all dynamic_deps (cc_binary|cc_test). def _separate_static_and_dynamic_link_libraries( + ctx, direct_children, - can_be_linked_dynamically, - transitive_dynamic_dep_labels = {}): + can_be_linked_dynamically): + ( + transitive_dynamic_dep_labels, + all_dynamic_dep_linker_inputs, + ) = _build_map_direct_dynamic_dep_to_transitive_dynamic_deps(ctx) + node = None all_children = reversed(direct_children) targets_to_be_linked_statically_map = {} @@ -209,7 +214,7 @@ def _separate_static_and_dynamic_link_libraries( transitive.append(first_owner_to_depset[child.owners[0]]) topologically_sorted_labels = depset(transitive = transitive, order = "topological").to_list() - return (targets_to_be_linked_statically_map, targets_to_be_linked_dynamically_set, topologically_sorted_labels) + return (targets_to_be_linked_statically_map, targets_to_be_linked_dynamically_set, topologically_sorted_labels, all_dynamic_dep_linker_inputs) def _create_linker_context(ctx, linker_inputs): return cc_common.create_linking_context( @@ -389,11 +394,15 @@ def _filter_inputs( # The targets_to_be_linked_statically_map points to whether the target to # be linked statically can be linked more than once. + # Entries in unused_dynamic_linker_inputs will be marked None if they are + # used ( targets_to_be_linked_statically_map, targets_to_be_linked_dynamically_set, topologically_sorted_labels, + unused_dynamic_linker_inputs, ) = _separate_static_and_dynamic_link_libraries( + ctx, graph_structure_aspect_nodes, can_be_linked_dynamically, ) @@ -437,6 +446,8 @@ def _filter_inputs( linker_inputs_seen[stringified_linker_input] = True owner = str(linker_input.owner) if owner in targets_to_be_linked_dynamically_set: + unused_dynamic_linker_inputs[transitive_exports[owner].owner] = None + # Link the library in this iteration dynamically, # transitive_exports contains the artifacts produced by a # cc_shared_library @@ -502,6 +513,8 @@ def _filter_inputs( message += dynamic_only_root + "\n" fail(message) + linker_inputs_count += _add_unused_dynamic_deps(ctx, unused_dynamic_linker_inputs, _add_linker_input_to_dict, topologically_sorted_labels, link_indirect_deps = False) + if ctx.attr.experimental_disable_topo_sort_do_not_use_remove_before_7_0: linker_inputs = experimental_remove_before_7_0_linker_inputs else: @@ -567,6 +580,39 @@ def _get_deps(ctx): return deps +def _build_map_direct_dynamic_dep_to_transitive_dynamic_deps(ctx): + all_dynamic_dep_linker_inputs = {} + direct_dynamic_dep_to_transitive_dynamic_deps = {} + for dep in ctx.attr.dynamic_deps: + owner = dep[CcSharedLibraryInfo].linker_input.owner + all_dynamic_dep_linker_inputs[owner] = dep[CcSharedLibraryInfo].linker_input + transitive_dynamic_dep_labels = [] + for dynamic_dep in dep[CcSharedLibraryInfo].dynamic_deps.to_list(): + all_dynamic_dep_linker_inputs[dynamic_dep[1].owner] = dynamic_dep[1] + transitive_dynamic_dep_labels.append(dynamic_dep[1].owner) + transitive_dynamic_dep_labels_set = depset(transitive_dynamic_dep_labels, order = "topological") + for export in dep[CcSharedLibraryInfo].exports: + direct_dynamic_dep_to_transitive_dynamic_deps[export] = transitive_dynamic_dep_labels_set + + return direct_dynamic_dep_to_transitive_dynamic_deps, all_dynamic_dep_linker_inputs + +def _add_unused_dynamic_deps(ctx, unused_dynamic_linker_inputs, add_linker_inputs_lambda, topologically_sorted_labels, link_indirect_deps): + linker_inputs_count = 0 + direct_dynamic_dep_labels = {dep[CcSharedLibraryInfo].linker_input.owner: True for dep in ctx.attr.dynamic_deps} + topologically_sorted_labels_set = {label: True for label in topologically_sorted_labels} + for dynamic_linker_input_owner, unused_linker_input in unused_dynamic_linker_inputs.items(): + should_link_input = (unused_linker_input and + (link_indirect_deps or dynamic_linker_input_owner in direct_dynamic_dep_labels)) + if should_link_input: + add_linker_inputs_lambda( + dynamic_linker_input_owner, + unused_dynamic_linker_inputs[dynamic_linker_input_owner], + ) + linker_inputs_count += 1 + if dynamic_linker_input_owner not in topologically_sorted_labels_set: + topologically_sorted_labels.append(dynamic_linker_input_owner) + return linker_inputs_count + def _cc_shared_library_impl(ctx): if not cc_common.check_experimental_cc_shared_library(): if len(ctx.attr.static_deps): @@ -806,3 +852,4 @@ build_exports_map_from_only_dynamic_deps = _build_exports_map_from_only_dynamic_ throw_linked_but_not_exported_errors = _throw_linked_but_not_exported_errors separate_static_and_dynamic_link_libraries = _separate_static_and_dynamic_link_libraries sort_linker_inputs = _sort_linker_inputs +add_unused_dynamic_deps = _add_unused_dynamic_deps diff --git a/src/main/starlark/builtins_bzl/common/exports.bzl b/src/main/starlark/builtins_bzl/common/exports.bzl index 2456597095678b..48d732bc0b38f9 100755 --- a/src/main/starlark/builtins_bzl/common/exports.bzl +++ b/src/main/starlark/builtins_bzl/common/exports.bzl @@ -72,7 +72,7 @@ exported_rules = { "java_lite_proto_library": java_lite_proto_library, "objc_import": objc_import, "objc_library": objc_library, - "-j2objc_library": j2objc_library, + "+j2objc_library": j2objc_library, "proto_library": proto_library, "cc_shared_library": cc_shared_library, "cc_binary": cc_binary, @@ -94,4 +94,6 @@ exported_to_java = { "get_cc_toolchain_provider": get_cc_toolchain_provider, "cc_toolchain_build_variables": cc_helper.cc_toolchain_build_variables, "apple_cc_toolchain_build_variables": objc_common.apple_cc_toolchain_build_variables, + "j2objc_mapping_file_info_union": objc_common.j2objc_mapping_file_info_union, + "j2objc_entry_class_info_union": objc_common.j2objc_entry_class_info_union, } diff --git a/src/main/starlark/builtins_bzl/common/objc/j2objc_aspect.bzl b/src/main/starlark/builtins_bzl/common/objc/j2objc_aspect.bzl index ddda3e0e8f422f..59cc4978dfccb0 100644 --- a/src/main/starlark/builtins_bzl/common/objc/j2objc_aspect.bzl +++ b/src/main/starlark/builtins_bzl/common/objc/j2objc_aspect.bzl @@ -57,7 +57,7 @@ def _proto_j2objc_source(ctx, proto_info, proto_sources, objc_file_path): def _get_output_objc_files(actions, srcs, objc_file_root_relative_path, suffix): objc_sources = [] for src in srcs: - src_path = src.short_path.removesuffix("." + src.extension) + src_path = src.path.removesuffix("." + src.extension) objc_source_path = paths.get_relative(objc_file_root_relative_path, src_path) + suffix objc_sources.append(actions.declare_file(objc_source_path)) return objc_sources @@ -97,7 +97,7 @@ def _java_j2objc_source(ctx, java_source_files, java_source_jars): header_tree_artifact_rel_path = _get_header_tree_artifact_rel_path(ctx) translated_header = ctx.actions.declare_directory(header_tree_artifact_rel_path) objc_hdrs.append(translated_header) - header_search_paths.append(translated_header.short_path) + header_search_paths.append(translated_header.path) return struct( target = ctx.label, @@ -239,7 +239,7 @@ def _create_j2objc_transpilation_action( dep_j2objc_mapping_file_provider, transitive_compile_time_jars, j2objc_source): - java_runtime = java_semantics.find_java_runtime_toolchain(ctx) + java_runtime = ctx.toolchains[java_semantics.JAVA_TOOLCHAIN_TYPE].java.java_runtime args = ctx.actions.args() args.use_param_file(param_file_arg = "@%s", use_always = True) @@ -285,7 +285,7 @@ def _create_j2objc_transpilation_action( args.add("--compiled_archive_file_path", compiled_library) boothclasspath_jar = ctx.file._jre_emul_jar - args.add("-Xbootclasspath:" + boothclasspath_jar.short_path) + args.add("-Xbootclasspath:" + boothclasspath_jar.path) module_files = ctx.attr._jre_emul_module.files.to_list() for file in module_files: @@ -304,17 +304,18 @@ def _create_j2objc_transpilation_action( args.add_all(java_source_files) + direct_files = [j2objc_deploy_jar, boothclasspath_jar] + if dead_code_report != None: + direct_files.append(dead_code_report) + if not experimental_j2objc_header_map: + direct_files.append(output_header_mapping_file) + ctx.actions.run( mnemonic = "TranspilingJ2objc", executable = ctx.executable._j2objc_wrapper, arguments = [args], inputs = depset( - [ - j2objc_deploy_jar, - boothclasspath_jar, - dead_code_report, - ] + module_files + java_source_files + java_source_jars + - [output_header_mapping_file] if not experimental_j2objc_header_map else [], + direct_files + module_files + java_source_files + java_source_jars, transitive = [ transitive_compile_time_jars, java_runtime.files, @@ -322,9 +323,8 @@ def _create_j2objc_transpilation_action( deps_class_mapping_files, ], ), - outputs = [output_dep_mapping_file, archive_source_mapping_file] + - j2objc_source.objc_srcs + - j2objc_source.objc_hdrs, + outputs = j2objc_source.objc_srcs + j2objc_source.objc_hdrs + + [output_dep_mapping_file, archive_source_mapping_file], toolchain = None, ) @@ -551,12 +551,14 @@ j2objc_aspect = aspect( default = "@" + cc_semantics.get_repo() + "//tools/j2objc:j2objc_header_map_binary", ), "_jre_emul_jar": attr.label( + cfg = "exec", allow_single_file = True, - default = Label("@//third_party/java/j2objc:jre_emul.jar"), + default = Label("@" + cc_semantics.get_repo() + "//third_party/java/j2objc:jre_emul.jar"), ), "_jre_emul_module": attr.label( + cfg = "exec", allow_files = True, - default = Label("@//third_party/java/j2objc:jre_emul_module"), + default = Label("@" + cc_semantics.get_repo() + "//third_party/java/j2objc:jre_emul_module"), ), "_dead_code_report": attr.label( allow_single_file = True, @@ -568,36 +570,17 @@ j2objc_aspect = aspect( ), "_jre_lib": attr.label( allow_files = True, - default = Label("@//third_party/java/j2objc:jre_core_lib"), - ), - "_xcrunwrapper": attr.label( - allow_files = True, - cfg = "exec", - default = "@" + cc_semantics.get_repo() + "//tools/objc:xcrunwrapper", - ), - "_xcode_config": attr.label( - allow_rules = ["xcode_config"], - default = configuration_field( - fragment = "apple", - name = "xcode_config_label", - ), - # TODO(kotlaja): Do we need "checkConstraints" here? Label doesn't have a flag attribute. - ), - "_zipper": attr.label( - allow_files = True, - cfg = "exec", - default = "@" + cc_semantics.get_repo() + "//tools/zip:zipper", + default = Label("@" + cc_semantics.get_repo() + "//third_party/java/j2objc:jre_core_lib"), ), "_j2objc_proto_toolchain": attr.label( default = configuration_field(fragment = "proto", name = "proto_toolchain_for_j2objc"), ), - "_java_toolchain_type": attr.label(default = java_semantics.JAVA_TOOLCHAIN_TYPE), "_cc_toolchain": attr.label( default = "@" + cc_semantics.get_repo() + "//tools/cpp:current_cc_toolchain", ), }, required_providers = [[JavaInfo], [ProtoInfo]], provides = [apple_common.Objc], - toolchains = [java_semantics.JAVA_TOOLCHAIN_TYPE, java_semantics.JAVA_RUNTIME_TOOLCHAIN_TYPE] + cc_helper.use_cpp_toolchain(), + toolchains = [java_semantics.JAVA_TOOLCHAIN_TYPE] + cc_helper.use_cpp_toolchain(), fragments = ["apple", "cpp", "j2objc", "objc", "proto"], ) diff --git a/src/main/starlark/builtins_bzl/common/objc/objc_common.bzl b/src/main/starlark/builtins_bzl/common/objc/objc_common.bzl index 65e283090a2c39..eb8c806102133d 100644 --- a/src/main/starlark/builtins_bzl/common/objc/objc_common.bzl +++ b/src/main/starlark/builtins_bzl/common/objc/objc_common.bzl @@ -15,6 +15,7 @@ """Common functionality for Objc rules.""" load(":common/cc/cc_info.bzl", "CcInfo") +load(":common/objc/providers.bzl", "J2ObjcEntryClassInfo", "J2ObjcMappingFileInfo") objc_internal = _builtins.internal.objc_internal apple_common = _builtins.toplevel.apple_common @@ -460,6 +461,37 @@ def _apple_cc_toolchain_build_variables(xcode_config): return apple_cc_toolchain_build_variables +# TODO(bazel-team): Delete this function when MultiArchBinarySupport is starlarkified. +def _j2objc_mapping_file_info_union(providers): + transitive_header_mapping_files = [] + transitive_class_mapping_files = [] + transitive_dependency_mapping_files = [] + transitive_archive_source_mapping_files = [] + + for provider in providers: + transitive_header_mapping_files.append(provider.header_mapping_files) + transitive_class_mapping_files.append(provider.class_mapping_files) + transitive_dependency_mapping_files.append(provider.dependency_mapping_files) + transitive_archive_source_mapping_files.append(provider.archive_source_mapping_files) + + return J2ObjcMappingFileInfo( + header_mapping_files = depset([], transitive = transitive_header_mapping_files), + class_mapping_files = depset([], transitive = transitive_class_mapping_files), + dependency_mapping_files = depset([], transitive = transitive_dependency_mapping_files), + archive_source_mapping_files = depset([], transitive = transitive_archive_source_mapping_files), + ) + +# TODO(bazel-team): Delete this function when MultiArchBinarySupport is starlarkified. +def _j2objc_entry_class_info_union(providers): + transitive_entry_classes = [] + + for provider in providers: + transitive_entry_classes.append(provider.entry_classes) + + return J2ObjcEntryClassInfo( + entry_classes = depset([], transitive = transitive_entry_classes), + ) + objc_common = struct( create_context_and_provider = _create_context_and_provider, to_string_with_minimum_components = _to_string_with_minimum_components, @@ -468,4 +500,6 @@ objc_common = struct( apple_cc_toolchain_build_variables = _apple_cc_toolchain_build_variables, is_apple_platform = _is_apple_platform, get_common_vars = _get_common_vars, + j2objc_mapping_file_info_union = _j2objc_mapping_file_info_union, + j2objc_entry_class_info_union = _j2objc_entry_class_info_union, ) diff --git a/src/main/starlark/builtins_bzl/common/objc/objc_library.bzl b/src/main/starlark/builtins_bzl/common/objc/objc_library.bzl index 1ea52415dd5eb7..038e7f84229c00 100644 --- a/src/main/starlark/builtins_bzl/common/objc/objc_library.bzl +++ b/src/main/starlark/builtins_bzl/common/objc/objc_library.bzl @@ -17,9 +17,10 @@ load("@_builtins//:common/cc/cc_helper.bzl", "cc_helper") load("@_builtins//:common/objc/compilation_support.bzl", "compilation_support") load("@_builtins//:common/objc/attrs.bzl", "common_attrs") -load("@_builtins//:common/objc/objc_common.bzl", "extensions") +load("@_builtins//:common/objc/objc_common.bzl", "extensions", "objc_common") load("@_builtins//:common/objc/semantics.bzl", "semantics") load("@_builtins//:common/objc/transitions.bzl", "apple_crosstool_transition") +load(":common/objc/providers.bzl", "J2ObjcEntryClassInfo", "J2ObjcMappingFileInfo") load(":common/cc/cc_info.bzl", "CcInfo") objc_internal = _builtins.internal.objc_internal @@ -78,7 +79,11 @@ def _objc_library_impl(ctx): compilation_support.validate_attributes(common_variables) - j2objc_providers = objc_internal.j2objc_providers_from_deps(ctx = ctx) + j2objc_mapping_file_infos = [dep[J2ObjcMappingFileInfo] for dep in ctx.attr.deps if J2ObjcMappingFileInfo in dep] + j2objc_mapping_file_info = objc_common.j2objc_mapping_file_info_union(providers = j2objc_mapping_file_infos) + + j2objc_entry_class_infos = [dep[J2ObjcEntryClassInfo] for dep in ctx.attr.deps if J2ObjcEntryClassInfo in dep] + j2objc_entry_class_info = objc_common.j2objc_entry_class_info_union(providers = j2objc_entry_class_infos) objc_provider = common_variables.objc_provider @@ -105,8 +110,8 @@ def _objc_library_impl(ctx): linking_context = linking_context, ), objc_provider, - j2objc_providers[0], - j2objc_providers[1], + j2objc_mapping_file_info, + j2objc_entry_class_info, instrumented_files_info, OutputGroupInfo(**output_groups), ] diff --git a/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/BUILD.builtin_test b/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/BUILD.builtin_test index 3df2f8b868258a..c3399ee3140806 100644 --- a/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/BUILD.builtin_test +++ b/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/BUILD.builtin_test @@ -25,7 +25,6 @@ licenses(["notice"]) package( default_visibility = ["//src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library:__subpackages__"], - features = ["exclude_bazel_rpaths_in_transitive_libs"], ) py_test( @@ -46,7 +45,6 @@ cc_binary( name = "binary", srcs = ["main.cc"], dynamic_deps = ["foo_so"], - features = ["use_rpath_instead_of_runpath"], deps = [ ":foo", ], @@ -63,7 +61,7 @@ cc_binary( "foo_so", "bar_so", ], - deps = ["foo", "bar"], + deps = ["foo"], ) cc_shared_library( @@ -79,32 +77,18 @@ cc_shared_library( deps = [":a_suffix"], ) -cc_library( - name = "diamond_lib1", - deps = [ - ":a_suffix", - ], -) - -cc_library( - name = "diamond_lib2", - deps = [ - ":a_suffix", - ], -) - cc_shared_library( name = "diamond_so", dynamic_deps = [":a_so"], features = ["windows_export_all_symbols"], - deps = [":qux", "diamond_lib1"], + deps = [":qux"], ) cc_shared_library( name = "diamond2_so", dynamic_deps = [":a_so"], features = ["windows_export_all_symbols"], - deps = [":bar", "diamond_lib2"], + deps = [":bar"], ) cc_binary( @@ -131,9 +115,10 @@ cc_shared_library( }), dynamic_deps = [ "bar_so", - "//src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library3:diff_pkg_so" + "//src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library3:diff_pkg_so", + "private_lib_so", ], - features = ["windows_export_all_symbols", "use_rpath_instead_of_runpath"], + features = ["windows_export_all_symbols"], exports_filter = [ ":indirect_dep2", ], @@ -447,6 +432,24 @@ cc_library( ], ) +cc_shared_library( + name = "private_lib_so", + deps = [ + ":private_lib", + ], +) + +genrule( + name = "private_cc_lib_source", + outs = ["private_cc_library.cc"], + cmd = "touch $@", +) + +cc_library( + name = "private_lib", + srcs = [":private_cc_library.cc"] +) + build_failure_test( name = "two_dynamic_deps_same_export_in_so_test", message = "Two shared libraries in dependencies export the same symbols", diff --git a/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/cc_shared_library_integration_test.sh b/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/cc_shared_library_integration_test.sh index 8bc6c1e88e7921..cf2769994b5c4a 100755 --- a/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/cc_shared_library_integration_test.sh +++ b/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/cc_shared_library_integration_test.sh @@ -52,7 +52,8 @@ function test_shared_library_symbols() { function test_shared_library_user_link_flags() { foo_so=$(find . -name libfoo_so.so) - objdump -x $foo_so | grep RPATH | grep "kittens" > /dev/null \ + # $RPATH defined in testenv.sh + objdump -x $foo_so | grep $RPATH | grep "kittens" > /dev/null \ || (echo "Expected to have RUNPATH contain 'kittens' (set by user_link_flags)" \ && exit 1) } diff --git a/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/failing_targets/BUILD.builtin_test b/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/failing_targets/BUILD.builtin_test index 1e4a63657e4703..5b0b495c8ed70f 100644 --- a/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/failing_targets/BUILD.builtin_test +++ b/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/failing_targets/BUILD.builtin_test @@ -12,7 +12,6 @@ cc_binary( dynamic_deps = ["//src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library:bar_so"], tags = TAGS, deps = [ - "//src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library:bar", "//src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library:barX", ], ) @@ -29,7 +28,6 @@ cc_shared_library( cc_library( name = "intermediate", deps = [ - "//src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library:bar", "//src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library:barX", ], ) diff --git a/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/starlark_tests.bzl b/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/starlark_tests.bzl index b04d475d7725e4..a74d65247cd9fd 100644 --- a/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/starlark_tests.bzl +++ b/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/starlark_tests.bzl @@ -58,6 +58,11 @@ def _linking_order_test_impl(env, target): matching.contains("foo.pic.o"), matching.contains("baz.pic.o"), ]).in_order() + + env.expect.that_collection(args).contains_at_least([ + "-lprivate_lib_so", + ]) + env.expect.where( detail = "liba_suffix.pic.o should be the last user library linked", ).that_str(user_libs[-1]).equals("a_suffix.pic.o") @@ -68,7 +73,7 @@ def _linking_order_test_impl(env, target): if "qux2.pic.o" in user_libs: found_bar = False for arg in args: - if "libbar" in arg: + if "-lbar_so" in arg: found_bar = True elif "qux2.pic.o" in arg: env.expect.where( @@ -181,6 +186,7 @@ def _runfiles_test_impl(env, target): "libfoo_so.so", "libbar_so.so", "libdiff_pkg_so.so", + "libprivate_lib_so.so", "Smain_Sstarlark_Stests_Sbuiltins_Ubzl_Scc_Scc_Ushared_Ulibrary_Stest_Ucc_Ushared_Ulibrary_Slibfoo_Uso.so", "Smain_Sstarlark_Stests_Sbuiltins_Ubzl_Scc_Scc_Ushared_Ulibrary_Stest_Ucc_Ushared_Ulibrary_Slibbar_Uso.so", "Smain_Sstarlark_Stests_Sbuiltins_Ubzl_Scc_Scc_Ushared_Ulibrary_Stest_Ucc_Ushared_Ulibrary3_Slibdiff_Upkg_Uso.so", diff --git a/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/testenv.sh b/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/testenv.sh index 1572f31ddafb68..5e946c88ca2052 100644 --- a/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/testenv.sh +++ b/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/testenv.sh @@ -14,3 +14,4 @@ # limitations under the License. LDD_BINARY="ldd" +RPATH="RUNPATH" diff --git a/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library3/BUILD.builtin_test b/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library3/BUILD.builtin_test index 7d15613e6746b2..39d541f8f37624 100644 --- a/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library3/BUILD.builtin_test +++ b/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library3/BUILD.builtin_test @@ -16,7 +16,7 @@ cc_library( cc_shared_library( name = "diff_pkg_so", - features = ["windows_export_all_symbols", "exclude_bazel_rpaths_in_transitive_libs"], + features = ["windows_export_all_symbols"], deps = [ ":diff_pkg", ], diff --git a/src/main/tools/process-tools-darwin.cc b/src/main/tools/process-tools-darwin.cc index 5843397f3aadf4..f2421823f03ef3 100644 --- a/src/main/tools/process-tools-darwin.cc +++ b/src/main/tools/process-tools-darwin.cc @@ -24,7 +24,9 @@ #include "src/main/tools/logging.h" #include "src/main/tools/process-tools.h" -int WaitForProcessToTerminate(pid_t pid) { +namespace { + +int WaitForProcessToTerminate(uintptr_t ident) { int kq; if ((kq = kqueue()) == -1) { return -1; @@ -34,7 +36,7 @@ int WaitForProcessToTerminate(pid_t pid) { // reports any pending such events, so this is not racy even if the // process happened to exit before we got to installing the kevent. struct kevent kc; - EV_SET(&kc, pid, EVFILT_PROC, EV_ADD | EV_ENABLE, NOTE_EXIT, 0, 0); + EV_SET(&kc, ident, EVFILT_PROC, EV_ADD | EV_ENABLE, NOTE_EXIT, 0, 0); int nev; struct kevent ke; @@ -48,9 +50,9 @@ int WaitForProcessToTerminate(pid_t pid) { if (nev != 1) { DIE("Expected only one event from the kevent call; got %d", nev); } - if (ke.ident != pid) { + if (ke.ident != ident) { DIE("Expected PID in the kevent to be %" PRIdMAX " but got %" PRIdMAX, - (intmax_t)pid, (intmax_t)ke.ident); + (intmax_t)ident, (intmax_t)ke.ident); } if (!(ke.fflags & NOTE_EXIT)) { DIE("Expected the kevent to be for an exit condition"); @@ -59,6 +61,16 @@ int WaitForProcessToTerminate(pid_t pid) { return close(kq); } +} // namespace + +int WaitForProcessToTerminate(pid_t pid) { + if (pid < 0) { + DIE("PID must be >= 0, got %" PRIdMAX, static_cast(pid)); + } + + return WaitForProcessToTerminate((uintptr_t)pid); +} + int WaitForProcessGroupToTerminate(pid_t pgid) { int name[] = {CTL_KERN, KERN_PROC, KERN_PROC_PGRP, pgid}; diff --git a/src/test/java/com/google/devtools/build/android/dexer/DexLimitTrackerTest.java b/src/test/java/com/google/devtools/build/android/dexer/DexLimitTrackerTest.java index f88ca36ab64709..9e93a0d9a67591 100644 --- a/src/test/java/com/google/devtools/build/android/dexer/DexLimitTrackerTest.java +++ b/src/test/java/com/google/devtools/build/android/dexer/DexLimitTrackerTest.java @@ -42,46 +42,61 @@ public void setUp() throws Exception { public void testUnderLimit() { DexLimitTracker tracker = new DexLimitTracker(Math.max(dex.methodIds().size(), dex.fieldIds().size())); - assertThat(tracker.track(dex)).isFalse(); + tracker.track(dex); + assertThat(tracker.outsideLimits()).isFalse(); } @Test public void testOverLimit() throws Exception { DexLimitTracker tracker = new DexLimitTracker(Math.max(dex.methodIds().size(), dex.fieldIds().size()) - 1); - assertThat(tracker.track(dex)).isTrue(); - assertThat(tracker.track(dex)).isTrue(); - assertThat(tracker.track(DexFiles.toDex(convertClass(DexLimitTracker.class)))).isTrue(); + tracker.track(dex); + assertThat(tracker.outsideLimits()).isTrue(); + tracker.track(dex); + assertThat(tracker.outsideLimits()).isTrue(); + tracker.track(DexFiles.toDex(convertClass(DexLimitTracker.class))); + assertThat(tracker.outsideLimits()).isTrue(); } @Test public void testRepeatedReferencesDeduped() throws Exception { DexLimitTracker tracker = new DexLimitTracker(Math.max(dex.methodIds().size(), dex.fieldIds().size())); - assertThat(tracker.track(dex)).isFalse(); - assertThat(tracker.track(dex)).isFalse(); - assertThat(tracker.track(dex)).isFalse(); - assertThat(tracker.track(dex)).isFalse(); - assertThat(tracker.track(DexFiles.toDex(convertClass(DexLimitTracker.class)))).isTrue(); - assertThat(tracker.track(dex)).isTrue(); + tracker.track(dex); + assertThat(tracker.outsideLimits()).isFalse(); + tracker.track(dex); + assertThat(tracker.outsideLimits()).isFalse(); + tracker.track(dex); + assertThat(tracker.outsideLimits()).isFalse(); + tracker.track(dex); + assertThat(tracker.outsideLimits()).isFalse(); + tracker.track(DexFiles.toDex(convertClass(DexLimitTracker.class))); + assertThat(tracker.outsideLimits()).isTrue(); + tracker.track(dex); + assertThat(tracker.outsideLimits()).isTrue(); } @Test public void testGoOverLimit() throws Exception { DexLimitTracker tracker = new DexLimitTracker(Math.max(dex.methodIds().size(), dex.fieldIds().size())); - assertThat(tracker.track(dex)).isFalse(); - assertThat(tracker.track(DexFiles.toDex(convertClass(DexLimitTracker.class)))).isTrue(); + tracker.track(dex); + assertThat(tracker.outsideLimits()).isFalse(); + tracker.track(DexFiles.toDex(convertClass(DexLimitTracker.class))); + assertThat(tracker.outsideLimits()).isTrue(); } @Test public void testClear() throws Exception { DexLimitTracker tracker = new DexLimitTracker(Math.max(dex.methodIds().size(), dex.fieldIds().size())); - assertThat(tracker.track(dex)).isFalse(); - assertThat(tracker.track(DexFiles.toDex(convertClass(DexLimitTracker.class)))).isTrue(); + tracker.track(dex); + assertThat(tracker.outsideLimits()).isFalse(); + tracker.track(DexFiles.toDex(convertClass(DexLimitTracker.class))); + assertThat(tracker.outsideLimits()).isTrue(); tracker.clear(); - assertThat(tracker.track(dex)).isFalse(); + tracker.track(dex); + assertThat(tracker.outsideLimits()).isFalse(); } private static DexFile convertClass(Class clazz) throws Exception { diff --git a/src/test/java/com/google/devtools/build/android/resources/RClassGeneratorTest.java b/src/test/java/com/google/devtools/build/android/resources/RClassGeneratorTest.java index 21cabeab64393e..8e1bc9cc746871 100644 --- a/src/test/java/com/google/devtools/build/android/resources/RClassGeneratorTest.java +++ b/src/test/java/com/google/devtools/build/android/resources/RClassGeneratorTest.java @@ -165,12 +165,18 @@ private void checkSimpleInts(boolean finalFields) throws Exception { "int id someTextView 0x7f080000", "int integer maxNotifications 0x7f090000", "int string alphabet 0x7f100000", - "int string ok 0x7f100001"); + "int string ok 0x7f100001", + // aapt2 link --package-id 0x80 produces IDs that are out of range of a java integer. + "int string largePackageId 0x80001000"); // R.txt for the library, where the values are not the final ones (so ignore them). We only use // this to keep the # of inner classes small (exactly the set needed by the library). ResourceSymbols symbolsInLibrary = createSymbolFile( - "lib.R.txt", "int attr agility 0x1", "int id someTextView 0x1", "int string ok 0x1"); + "lib.R.txt", + "int attr agility 0x1", + "int id someTextView 0x1", + "int string ok 0x1", + "int string largePackageId 0x1"); Path out = temp.resolve("classes"); Files.createDirectories(out); RClassGenerator writer = RClassGenerator.with(out, symbolValues.asInitializers(), finalFields); @@ -198,7 +204,7 @@ private void checkSimpleInts(boolean finalFields) throws Exception { out, "com.bar.R$string", outerClass, - ImmutableMap.of("ok", 0x7f100001), + ImmutableMap.of("ok", 0x7f100001, "largePackageId", 0x80001000), ImmutableMap.>of(), finalFields); } diff --git a/src/test/java/com/google/devtools/build/lib/analysis/AutoExecGroupsTest.java b/src/test/java/com/google/devtools/build/lib/analysis/AutoExecGroupsTest.java index fc3b8ac43655f5..2344f17117ff5d 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/AutoExecGroupsTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/AutoExecGroupsTest.java @@ -1399,8 +1399,6 @@ public void javaCommonCompile_automaticExecGroupsDisabled_lazyActionExecutesOnSe " output = output_jar,", " java_toolchain = ctx.toolchains['" + TestConstants.JAVA_TOOLCHAIN_TYPE + "'].java,", " resources = ctx.files.resources,", - " resource_jars = ctx.files.resource_jars,", - " classpath_resources = ctx.files.classpath_resources,", " )", " return [java_info, DefaultInfo(files = depset([output_jar]))]", "custom_rule = rule(", @@ -1408,8 +1406,6 @@ public void javaCommonCompile_automaticExecGroupsDisabled_lazyActionExecutesOnSe " toolchains = ['//rule:toolchain_type_2', '" + TestConstants.JAVA_TOOLCHAIN_TYPE + "'],", " attrs = {", " 'resources': attr.label_list(allow_files = True),", - " 'resource_jars': attr.label_list(allow_files = True),", - " 'classpath_resources': attr.label_list(allow_files = True),", " },", " provides = [JavaInfo],", " fragments = ['java']", @@ -1417,8 +1413,7 @@ public void javaCommonCompile_automaticExecGroupsDisabled_lazyActionExecutesOnSe scratch.file( "bazel_internal/test/BUILD", "load('//bazel_internal/test:defs.bzl', 'custom_rule')", - "custom_rule(name = 'custom_rule_name', resources = ['Resources.java'], resource_jars =" - + " ['ResourceJars.java'], classpath_resources = ['ClasspathResources.java'])"); + "custom_rule(name = 'custom_rule_name', resources = ['Resources.java'])"); useConfiguration( "--incompatible_auto_exec_groups", "--experimental_turbine_annotation_processing"); @@ -1446,8 +1441,6 @@ public void javaCommonCompile_automaticExecGroupsDisabled_lazyActionExecutesOnSe " output = output_jar,", " java_toolchain = ctx.toolchains['" + TestConstants.JAVA_TOOLCHAIN_TYPE + "'].java,", " resources = ctx.files.resources,", - " resource_jars = ctx.files.resource_jars,", - " classpath_resources = ctx.files.classpath_resources,", " )", " return [java_info, DefaultInfo(files = depset([output_jar]))]", "custom_rule = rule(", @@ -1455,8 +1448,6 @@ public void javaCommonCompile_automaticExecGroupsDisabled_lazyActionExecutesOnSe " toolchains = ['//rule:toolchain_type_2', '" + TestConstants.JAVA_TOOLCHAIN_TYPE + "'],", " attrs = {", " 'resources': attr.label_list(allow_files = True),", - " 'resource_jars': attr.label_list(allow_files = True),", - " 'classpath_resources': attr.label_list(allow_files = True),", " },", " provides = [JavaInfo],", " fragments = ['java']", @@ -1464,8 +1455,7 @@ public void javaCommonCompile_automaticExecGroupsDisabled_lazyActionExecutesOnSe scratch.file( "bazel_internal/test/BUILD", "load('//bazel_internal/test:defs.bzl', 'custom_rule')", - "custom_rule(name = 'custom_rule_name', resources = ['Resources.java'], resource_jars =" - + " ['ResourceJars.java'], classpath_resources = ['ClasspathResources.java'])"); + "custom_rule(name = 'custom_rule_name', resources = ['Resources.java'])"); useConfiguration("--experimental_turbine_annotation_processing"); ImmutableList actions = getActions("//bazel_internal/test:custom_rule_name"); diff --git a/src/test/java/com/google/devtools/build/lib/analysis/BUILD b/src/test/java/com/google/devtools/build/lib/analysis/BUILD index 9c4dff27ffa6a8..aa6e0028a31696 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/BUILD +++ b/src/test/java/com/google/devtools/build/lib/analysis/BUILD @@ -101,6 +101,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:config/transitions/transition_collector", "//src/main/java/com/google/devtools/build/lib/analysis:config/transitions/transition_factory", "//src/main/java/com/google/devtools/build/lib/analysis:configured_target", + "//src/main/java/com/google/devtools/build/lib/analysis:configured_target_value", "//src/main/java/com/google/devtools/build/lib/analysis:dependency", "//src/main/java/com/google/devtools/build/lib/analysis:dependency_key", "//src/main/java/com/google/devtools/build/lib/analysis:dependency_kind", diff --git a/src/test/java/com/google/devtools/build/lib/analysis/ConfigurableAttributesTest.java b/src/test/java/com/google/devtools/build/lib/analysis/ConfigurableAttributesTest.java index ef6be77deebae6..280593f8edbf3d 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/ConfigurableAttributesTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/ConfigurableAttributesTest.java @@ -13,6 +13,7 @@ // limitations under the License. package com.google.devtools.build.lib.analysis; +import static com.google.common.testing.GcFinalization.awaitClear; import static com.google.common.truth.Truth.assertThat; import static com.google.devtools.build.lib.packages.Attribute.attr; import static com.google.devtools.build.lib.packages.BuildType.LABEL_LIST; @@ -32,9 +33,12 @@ import com.google.devtools.build.lib.packages.RuleClass.ToolchainResolutionMode; import com.google.devtools.build.lib.packages.Type; import com.google.devtools.build.lib.skyframe.ConfiguredTargetAndData; +import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey; +import com.google.devtools.build.lib.skyframe.SkyframeExecutorWrappingWalkableGraph; import com.google.devtools.build.lib.testutil.TestRuleClassProvider; import com.google.devtools.build.lib.util.FileTypeSet; import java.io.IOException; +import java.lang.ref.WeakReference; import java.util.Collection; import java.util.Set; import org.junit.Before; @@ -1927,4 +1931,71 @@ public void selectWithLabelKeysInMacro() throws Exception { /*expected:*/ ImmutableList.of("bin java/foo/libb.jar", "bin java/foo/libb2.jar"), /*not expected:*/ ImmutableList.of("bin java/foo/liba.jar", "bin java/foo/liba2.jar")); } + + @Test + public void proxyKeysAreRetained() throws Exception { + // This test case verifies that when a ProxyConfiguredTargetKey is created, it is retained. + scratch.file( + "conditions/BUILD", + "constraint_setting(name = 'animal')", + "constraint_value(name = 'manatee', constraint_setting = 'animal')", + "constraint_value(name = 'koala', constraint_setting = 'animal')", + "platform(", + " name = 'manatee_platform',", + " constraint_values = [':manatee'],", + ")", + "platform(", + " name = 'koala_platform',", + " constraint_values = [':koala'],", + ")"); + scratch.file( + "check/BUILD", + "filegroup(name = 'adep', srcs = ['afile'])", + "filegroup(name = 'bdep', srcs = ['bfile'])", + "filegroup(name = 'hello',", + " srcs = select({", + " '//conditions:manatee': [':adep'],", + " '//conditions:koala': [':bdep'],", + " }))"); + + useConfiguration("--experimental_platforms=//conditions:manatee_platform"); + ConfiguredTarget hello = getConfiguredTarget("//check:hello"); + + var koalaLabel = Label.parseCanonical("//conditions:koala"); + + // Shakes the interner to try to get any non-strongly reachable keys to fall out. This should + // cause the ProxyConfiguredTargetKey created for "//conditions:koala" to fall out if it's not + // otherwise retained. + // + // Creates and inserts a canary key into the interner that can be used to detect eviction of + // weak keys. + var canaryKey = new WeakReference<>(ConfiguredTargetKey.builder().setLabel(koalaLabel).build()); + awaitClear(canaryKey); + // Once we get here we know that the canaryKey is no longer in the weak interner. Due to the + // collection properties of weak references, that implies the interner now has no weakly + // reachable keys at all. + + // Since //conditions:koala is a ConfigCondition, so it would be requested by //check:hello + // using //check:hello's configuration. + var koalaOwner = + ConfiguredTargetKey.builder() + .setLabel(koalaLabel) + .setConfigurationKey(hello.getConfigurationKey()) + .build(); + // Uses a WalkableGraph lookup to ensure there is an existing //conditions:koala instance that + // was created using koalaOwner. + var walkableGraph = SkyframeExecutorWrappingWalkableGraph.of(skyframeExecutor); + var koala = (ConfiguredTargetValue) walkableGraph.getValue(koalaOwner.toKey()); + assertThat(koala).isNotNull(); + + // constraint_value has a NoConfigTransition rule transition so a corresponding proxy key + // should exist. + ConfiguredTargetKey koalaKey = + ConfiguredTargetKey.builder() + .setLabel(koalaLabel) + .setConfigurationKey(koala.getConfiguredTarget().getConfigurationKey()) + .build(); + assertThat(koalaKey.isProxy()).isTrue(); + assertThat(koalaKey.toKey()).isEqualTo(koalaOwner); + } } diff --git a/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java b/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java index 637a60ae7c32c3..67802e376c768a 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java @@ -26,6 +26,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; +import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsUtil; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; @@ -64,8 +65,7 @@ protected ImmutableList extraPrecomputedValues() throws Exception { PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF), - PrecomputedValue.injected( - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of())); + PrecomputedValue.injected(YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, ImmutableList.of())); } @Override diff --git a/src/test/java/com/google/devtools/build/lib/analysis/SourceManifestActionTest.java b/src/test/java/com/google/devtools/build/lib/analysis/SourceManifestActionTest.java index a8aa13bdf98c54..a6b4e4f17eafd3 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/SourceManifestActionTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/SourceManifestActionTest.java @@ -163,18 +163,20 @@ public boolean isRemotable() { @Test public void testManifestWriterIntegration() throws Exception { MockManifestWriter mockWriter = new MockManifestWriter(); - new SourceManifestAction( - mockWriter, - NULL_ACTION_OWNER, - manifestOutputFile, - new Runfiles.Builder("TESTING", false).addSymlinks(fakeManifest).build()) - .getFileContentsAsString(reporter); + String manifestContents = + new SourceManifestAction( + mockWriter, + NULL_ACTION_OWNER, + manifestOutputFile, + new Runfiles.Builder("TESTING", false).addSymlinks(fakeManifest).build()) + .getFileContents(reporter); assertThat(mockWriter.unconsumedInputs()).isEqualTo(0); + assertThat(manifestContents).isEmpty(); } @Test public void testSimpleFileWriting() throws Exception { - String manifestContents = createSymlinkAction().getFileContentsAsString(reporter); + String manifestContents = createSymlinkAction().getFileContents(reporter); assertThat(manifestContents) .isEqualTo( "TESTING/trivial/BUILD /workspace/trivial/BUILD\n" @@ -188,7 +190,7 @@ public void testSimpleFileWriting() throws Exception { */ @Test public void testSourceOnlyFormatting() throws Exception { - String manifestContents = createSourceOnlyAction().getFileContentsAsString(reporter); + String manifestContents = createSourceOnlyAction().getFileContents(reporter); assertThat(manifestContents) .isEqualTo( "TESTING/trivial/BUILD\n" @@ -207,7 +209,7 @@ public void testSwigLibrariesTriggerInitDotPyInclusion() throws Exception { Path swiggedFile = scratch.file("swig/fakeLib.so"); Artifact swigDotSO = ActionsTestUtil.createArtifact(swiggedLibPath, swiggedFile); fakeManifest.put(swiggedFile.relativeTo(rootDirectory), swigDotSO); - String manifestContents = createSymlinkAction().getFileContentsAsString(reporter); + String manifestContents = createSymlinkAction().getFileContents(reporter); assertThat(manifestContents).containsMatch(".*TESTING/swig/__init__.py .*"); assertThat(manifestContents).containsMatch("fakeLib.so"); } @@ -219,7 +221,7 @@ public void testNoPythonOrSwigLibrariesDoNotTriggerInitDotPyInclusion() throws E Path nonPythonFile = scratch.file("not_python/blob_of_data"); Artifact nonPython = ActionsTestUtil.createArtifact(nonPythonPath, nonPythonFile); fakeManifest.put(nonPythonFile.relativeTo(rootDirectory), nonPython); - String manifestContents = createSymlinkAction().getFileContentsAsString(reporter); + String manifestContents = createSymlinkAction().getFileContents(reporter); assertThat(manifestContents).doesNotContain("not_python/__init__.py \n"); assertThat(manifestContents).containsMatch("blob_of_data"); } @@ -367,7 +369,7 @@ public void testUnresolvedSymlink() throws Exception { assertThat(inputs).isEqualTo(action.getInputs()); assertThat(inputs.toList()).isEqualTo(action.getInputs().toList()); - assertThat(action.getFileContentsAsString(reporter)) + assertThat(action.getFileContents(reporter)) .isEqualTo( "TESTING/BUILD /workspace/trivial/BUILD\n" + "TESTING/absolute_symlink /absolute/path\n" diff --git a/src/test/java/com/google/devtools/build/lib/analysis/StarlarkRuleTransitionProviderTest.java b/src/test/java/com/google/devtools/build/lib/analysis/StarlarkRuleTransitionProviderTest.java index 2cba454ff7dead..e53fdd5054ea7e 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/StarlarkRuleTransitionProviderTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/StarlarkRuleTransitionProviderTest.java @@ -29,6 +29,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; +import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsUtil; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; @@ -71,8 +72,7 @@ protected ImmutableList extraPrecomputedValues() { ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl())), PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false), PrecomputedValue.injected(ModuleFileFunction.MODULE_OVERRIDES, ImmutableMap.of()), - PrecomputedValue.injected( - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), + PrecomputedValue.injected(YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), PrecomputedValue.injected( BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( diff --git a/src/test/java/com/google/devtools/build/lib/analysis/mock/cc_toolchain_config.bzl b/src/test/java/com/google/devtools/build/lib/analysis/mock/cc_toolchain_config.bzl index c7be375090d0f9..0d25ecab4cd7ca 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/mock/cc_toolchain_config.bzl +++ b/src/test/java/com/google/devtools/build/lib/analysis/mock/cc_toolchain_config.bzl @@ -54,6 +54,7 @@ _FEATURE_NAMES = struct( user_compile_flags = "user_compile_flags", thin_lto = "thin_lto", no_use_lto_indexing_bitcode_file = "no_use_lto_indexing_bitcode_file", + use_lto_native_object_directory = "use_lto_native_object_directory", thin_lto_linkstatic_tests_use_shared_nonlto_backends = "thin_lto_linkstatic_tests_use_shared_nonlto_backends", thin_lto_all_linkstatic_use_shared_nonlto_backends = "thin_lto_all_linkstatic_use_shared_nonlto_backends", enable_afdo_thinlto = "enable_afdo_thinlto", @@ -475,6 +476,9 @@ _no_use_lto_indexing_bitcode_file_feature = feature( name = _FEATURE_NAMES.no_use_lto_indexing_bitcode_file, ) +_use_lto_native_object_directory_feature = feature( + name = _FEATURE_NAMES.use_lto_native_object_directory, +) _thin_lto_feature = feature( name = _FEATURE_NAMES.thin_lto, flag_sets = [ @@ -1322,6 +1326,7 @@ _feature_name_to_feature = { _FEATURE_NAMES.user_compile_flags: _user_compile_flags_feature, _FEATURE_NAMES.thin_lto: _thin_lto_feature, _FEATURE_NAMES.no_use_lto_indexing_bitcode_file: _no_use_lto_indexing_bitcode_file_feature, + _FEATURE_NAMES.use_lto_native_object_directory: _use_lto_native_object_directory_feature, _FEATURE_NAMES.thin_lto_linkstatic_tests_use_shared_nonlto_backends: _thin_lto_linkstatic_tests_use_shared_nonlto_backends_feature, _FEATURE_NAMES.thin_lto_all_linkstatic_use_shared_nonlto_backends: _thin_lto_all_linkstatic_use_shared_nonlto_backends_feature, _FEATURE_NAMES.enable_afdo_thinlto: _enable_afdo_thinlto_feature, diff --git a/src/test/java/com/google/devtools/build/lib/analysis/select/AbstractAttributeMapperTest.java b/src/test/java/com/google/devtools/build/lib/analysis/select/AbstractAttributeMapperTest.java index b9e5d1221cb9f9..3cc57c8789f456 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/select/AbstractAttributeMapperTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/select/AbstractAttributeMapperTest.java @@ -20,7 +20,6 @@ import com.google.devtools.build.lib.packages.AbstractAttributeMapper; import com.google.devtools.build.lib.packages.AttributeMap; import com.google.devtools.build.lib.packages.BuildType; -import com.google.devtools.build.lib.packages.Package; import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.packages.Type; import java.util.ArrayList; @@ -62,10 +61,7 @@ public void testPackageDefaultProperties() throws Exception { rule = scratchRule("a", "myrule", "cc_binary(name = 'myrule',", " srcs = ['a', 'b', 'c'])"); - Package pkg = rule.getPackage(); - assertThat(mapper.getPackageDefaultHdrsCheck()).isEqualTo(pkg.getDefaultHdrsCheck()); - assertThat(mapper.getPackageDefaultTestOnly()).isEqualTo(pkg.getDefaultTestOnly()); - assertThat(mapper.getPackageDefaultDeprecation()).isEqualTo(pkg.getDefaultDeprecation()); + assertThat(mapper.getPackageArgs()).isEqualTo(rule.getPackage().getPackageArgs()); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java index 9d1b863d90a029..a2672c0eafba21 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java @@ -47,6 +47,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; +import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsUtil; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; @@ -250,7 +251,7 @@ protected void useRuleClassProvider(ConfiguredRuleClassProvider ruleClassProvide BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), + YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), @@ -301,7 +302,7 @@ private void reinitializeSkyframeExecutor() { BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), + YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.WARNING), diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewForTesting.java b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewForTesting.java index 8b2ad5419fb708..91e7395e9688ff 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewForTesting.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewForTesting.java @@ -425,7 +425,7 @@ private OrderedSetMultimap getPrerequis eventHandler, new PrerequisiteParameters( ConfiguredTargetKey.fromConfiguredTarget(target), - state.targetAndConfiguration.getTarget(), + state.targetAndConfiguration.getTarget().getAssociatedRule(), /* aspects= */ ImmutableList.of(), skyframeBuildView.getStarlarkTransitionCache(), toolchainContexts, diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java index e64e0bef52605b..120283f50f273a 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java @@ -148,7 +148,7 @@ public void setup() throws Exception { BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); + YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java index 4efcebc1ece290..2388c6f9498759 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java @@ -203,7 +203,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of()); ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, true); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); + YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( @@ -283,7 +283,7 @@ public void moduleWithFlags() throws Exception { ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, true); ModuleFileFunction.REGISTRIES.set(differencer, registries); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of("my_dep_1", override)); - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, yankedVersions); + YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, yankedVersions); BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( differencer, CheckDirectDepsMode.ERROR); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java index 47d15dcc2e1e65..f240b5f30cf273 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java @@ -131,7 +131,7 @@ public void setup() throws Exception { BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); + YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); } @Test @@ -282,7 +282,7 @@ public void testYankedVersionCheckSuccess() throws Exception { @Test public void testYankedVersionCheckIgnoredByAll() throws Exception { setupModulesForYankedVersion(); - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of("all")); + YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of("all")); EvaluationResult result = evaluator.evaluate(ImmutableList.of(BazelModuleResolutionValue.KEY), evaluationContext); assertThat(result.hasError()).isFalse(); @@ -291,8 +291,7 @@ public void testYankedVersionCheckIgnoredByAll() throws Exception { @Test public void testYankedVersionCheckIgnoredBySpecific() throws Exception { setupModulesForYankedVersion(); - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set( - differencer, ImmutableList.of("b@1.0")); + YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of("b@1.0")); EvaluationResult result = evaluator.evaluate(ImmutableList.of(BazelModuleResolutionValue.KEY), evaluationContext); assertThat(result.hasError()).isFalse(); @@ -301,8 +300,7 @@ public void testYankedVersionCheckIgnoredBySpecific() throws Exception { @Test public void testBadYankedVersionFormat() throws Exception { setupModulesForYankedVersion(); - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set( - differencer, ImmutableList.of("b~1.0")); + YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of("b~1.0")); EvaluationResult result = evaluator.evaluate(ImmutableList.of(BazelModuleResolutionValue.KEY), evaluationContext); assertThat(result.hasError()).isTrue(); @@ -329,4 +327,86 @@ private void setupModulesForYankedVersion() throws Exception { .addYankedVersion("b", ImmutableMap.of(Version.parse("1.0"), "1.0 is a bad version!")); ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); } + + @Test + public void testYankedVersionSideEffects_equalCompatibilityLevel() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='mod', version='1.0')", + "bazel_dep(name = 'a', version = '1.0')", + "bazel_dep(name = 'b', version = '1.1')"); + + FakeRegistry registry = + registryFactory + .newFakeRegistry("/bar") + .addModule( + createModuleKey("a", "1.0"), + "module(name='a', version='1.0')", + "bazel_dep(name='b', version='1.0')") + .addModule(createModuleKey("c", "1.0"), "module(name='c', version='1.0')") + .addModule(createModuleKey("c", "1.1"), "module(name='c', version='1.1')") + .addModule( + createModuleKey("b", "1.0"), + "module(name='b', version='1.0', compatibility_level = 2)", + "bazel_dep(name='c', version='1.1')", + "print('hello from yanked version')") + .addModule( + createModuleKey("b", "1.1"), + "module(name='b', version='1.1', compatibility_level = 2)", + "bazel_dep(name='c', version='1.0')") + .addYankedVersion("b", ImmutableMap.of(Version.parse("1.0"), "1.0 is a bad version!")); + + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(BazelModuleResolutionValue.KEY), evaluationContext); + + assertThat(result.hasError()).isFalse(); + assertThat(result.get(BazelModuleResolutionValue.KEY).getResolvedDepGraph().keySet()) + .containsExactly( + ModuleKey.ROOT, + createModuleKey("a", "1.0"), + createModuleKey("b", "1.1"), + createModuleKey("c", "1.0")); + assertDoesNotContainEvent("hello from yanked version"); + } + + @Test + public void testYankedVersionSideEffects_differentCompatibilityLevel() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='mod', version='1.0')", + "bazel_dep(name = 'a', version = '1.0')", + "bazel_dep(name = 'b', version = '1.1')"); + + FakeRegistry registry = + registryFactory + .newFakeRegistry("/bar") + .addModule( + createModuleKey("a", "1.0"), + "module(name='a', version='1.0')", + "bazel_dep(name='b', version='1.0')") + .addModule(createModuleKey("c", "1.0"), "module(name='c', version='1.0')") + .addModule(createModuleKey("c", "1.1"), "module(name='c', version='1.1')") + .addModule( + createModuleKey("b", "1.0"), + "module(name='b', version='1.0', compatibility_level = 2)", + "bazel_dep(name='c', version='1.1')", + "print('hello from yanked version')") + .addModule( + createModuleKey("b", "1.1"), + "module(name='b', version='1.1', compatibility_level = 3)", + "bazel_dep(name='c', version='1.0')") + .addYankedVersion("b", ImmutableMap.of(Version.parse("1.0"), "1.0 is a bad version!")); + + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(BazelModuleResolutionValue.KEY), evaluationContext); + + assertThat(result.hasError()).isTrue(); + assertThat(result.getError().toString()) + .contains( + "a@1.0 depends on b@1.0 with compatibility level 2, but depends on b@1.1 with" + + " compatibility level 3 which is different"); + assertDoesNotContainEvent("hello from yanked version"); + } } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java index 190521f59625ab..8e100021b95972 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java @@ -140,7 +140,7 @@ public void setup() throws Exception { ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of()); ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, false); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); + YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( differencer, CheckDirectDepsMode.WARNING); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/DiscoveryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/DiscoveryTest.java index fda410324d68c6..47aca2e2b01d55 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/DiscoveryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/DiscoveryTest.java @@ -41,6 +41,7 @@ import com.google.devtools.build.lib.rules.repository.RepositoryFunction; import com.google.devtools.build.lib.skyframe.BazelSkyframeExecutorConstants; import com.google.devtools.build.lib.skyframe.BzlmodRepoRuleFunction; +import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction; import com.google.devtools.build.lib.skyframe.ExternalFilesHelper; import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction; import com.google.devtools.build.lib.skyframe.FileFunction; @@ -178,6 +179,9 @@ private void setUpWithBuiltinModules(ImmutableMap b .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, new BzlmodRepoRuleFunction(ruleClassProvider, directories)) + .put( + SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, + new ClientEnvironmentFunction(new AtomicReference<>(ImmutableMap.of()))) .buildOrThrow(), differencer); @@ -196,7 +200,7 @@ private void setUpWithBuiltinModules(ImmutableMap b RepositoryDelegatorFunction.RESOLVED_FILE_FOR_VERIFICATION.set(differencer, Optional.empty()); ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, false); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); + YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java index 134a7e1b4636b0..73cf61c2bb7461 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java @@ -284,7 +284,7 @@ public void setup() throws Exception { RepositoryDelegatorFunction.RESOLVED_FILE_FOR_VERIFICATION.set(differencer, Optional.empty()); ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, false); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); + YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( differencer, CheckDirectDepsMode.WARNING); diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java index d04da12c560e00..8ff80a284e55f5 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java @@ -43,6 +43,7 @@ import com.google.devtools.build.lib.rules.repository.RepositoryFunction; import com.google.devtools.build.lib.skyframe.BazelSkyframeExecutorConstants; import com.google.devtools.build.lib.skyframe.BzlmodRepoRuleFunction; +import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction; import com.google.devtools.build.lib.skyframe.ExternalFilesHelper; import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction; import com.google.devtools.build.lib.skyframe.FileFunction; @@ -152,6 +153,9 @@ private void setUpWithBuiltinModules(ImmutableMap b .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, new BzlmodRepoRuleFunction(ruleClassProvider, directories)) + .put( + SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, + new ClientEnvironmentFunction(new AtomicReference<>(ImmutableMap.of()))) .buildOrThrow(), differencer); @@ -170,7 +174,7 @@ private void setUpWithBuiltinModules(ImmutableMap b RepositoryDelegatorFunction.RESOLVED_FILE_FOR_VERIFICATION.set(differencer, Optional.empty()); ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, false); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); + YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProviderTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProviderTest.java index 60cbc8a9e593cf..87770babb5b860 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProviderTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProviderTest.java @@ -147,11 +147,6 @@ public void objcConsistency() { checkModule(ObjcRules.INSTANCE); } - @Test - public void j2objcConsistency() { - checkModule(J2ObjcRules.INSTANCE); - } - @Test public void variousWorkspaceConsistency() { checkModule(BazelRuleClassProvider.VARIOUS_WORKSPACE_RULES); diff --git a/src/test/java/com/google/devtools/build/lib/buildtool/util/BuildIntegrationTestCase.java b/src/test/java/com/google/devtools/build/lib/buildtool/util/BuildIntegrationTestCase.java index c353474f596b1a..db8f2db1609408 100644 --- a/src/test/java/com/google/devtools/build/lib/buildtool/util/BuildIntegrationTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/buildtool/util/BuildIntegrationTestCase.java @@ -144,6 +144,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Predicate; import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; import org.junit.After; @@ -369,9 +370,28 @@ public final void cleanUp() throws Exception { LoggingUtil.installRemoteLoggerForTesting(null); if (OS.getCurrent() == OS.WINDOWS) { - // Bazel runtime still holds the file handle of windows_jni.dll making it impossible to delete - // on Windows. Try to delete all other files (and directories). - bestEffortDeleteTreesBelow(testRoot, "windows_jni.dll"); + bestEffortDeleteTreesBelow( + testRoot, + filename -> { + // Bazel runtime still holds the file handle of windows_jni.dll making it impossible to + // delete on Windows. + if (filename.equals("windows_jni.dll")) { + return true; + } + + // mockito's inline mock maker manipulates byte code of mocked methods and output new + // byte code in a temporary jarfile with pattern mockitobootXXXXXXX.jar. It then loads + // these jar files into JVM to make mock effective which means Bazel runtime still holds + // handles of these files making it impossible to delete on Windows. + // + // See https://github.com/mockito/mockito/issues/1379#issuecomment-466372914 and + // https://github.com/mockito/mockito/blob/91f18ea1648e389bea06289d818def7978e82288/src/main/java/org/mockito/internal/creation/bytebuddy/InlineDelegateByteBuddyMockMaker.java#L123C10-L123C10. + if (filename.startsWith("mockitoboot") && filename.endsWith(".jar")) { + return true; + } + + return false; + }); } else { testRoot.deleteTreesBelow(); // (comment out during debugging) } @@ -382,7 +402,8 @@ public final void cleanUp() throws Exception { Thread.interrupted(); // If there was a crash in test case, main thread was interrupted. } - private static void bestEffortDeleteTreesBelow(Path path, String canSkip) throws IOException { + private static void bestEffortDeleteTreesBelow(Path path, Predicate canSkip) + throws IOException { for (Dirent dirent : path.readdir(Symlinks.NOFOLLOW)) { Path child = path.getRelative(dirent.getName()); if (dirent.getType() == Dirent.Type.DIRECTORY) { @@ -396,7 +417,7 @@ private static void bestEffortDeleteTreesBelow(Path path, String canSkip) throws try { child.delete(); } catch (IOException e) { - if (!child.getBaseName().equals(canSkip)) { + if (!canSkip.test(child.getBaseName())) { throw e; } } diff --git a/src/test/java/com/google/devtools/build/lib/cmdline/BUILD b/src/test/java/com/google/devtools/build/lib/cmdline/BUILD index 4c67386a6584b6..94ea4e6b45f90b 100644 --- a/src/test/java/com/google/devtools/build/lib/cmdline/BUILD +++ b/src/test/java/com/google/devtools/build/lib/cmdline/BUILD @@ -50,6 +50,7 @@ java_test( java_test( name = "LabelInternerIntegrationTest", srcs = ["LabelInternerIntegrationTest.java"], + jvm_flags = ["-DBAZEL_USE_POOLED_LABEL_INTERNER=1"], deps = [ "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/concurrent", diff --git a/src/test/java/com/google/devtools/build/lib/events/util/EventCollectionApparatus.java b/src/test/java/com/google/devtools/build/lib/events/util/EventCollectionApparatus.java index 888153751d1490..cc734fb69207b1 100644 --- a/src/test/java/com/google/devtools/build/lib/events/util/EventCollectionApparatus.java +++ b/src/test/java/com/google/devtools/build/lib/events/util/EventCollectionApparatus.java @@ -191,14 +191,6 @@ public Event assertContainsWarning(String expectedMessage) { return MoreAsserts.assertContainsEvent(eventCollector, expectedMessage, EventKind.WARNING); } - /** - * Utility method: Assert that the {@link #collector()} has received a - * debug message with the {@code expectedMessage}. - */ - public Event assertContainsDebug(String expectedMessage) { - return MoreAsserts.assertContainsEvent(eventCollector, expectedMessage, EventKind.DEBUG); - } - /** * Utility method: Assert that the {@link #collector()} has received an event of the given type * and with the {@code expectedMessage}. @@ -213,16 +205,6 @@ public List assertContainsEventWithFrequency(String expectedMessage, expectedFrequency); } - /** - * Utility method: Assert that the {@link #collector()} has received an - * event with the {@code expectedMessage} in quotes. - */ - - public Event assertContainsEventWithWordsInQuotes(String... words) { - return MoreAsserts.assertContainsEventWithWordsInQuotes( - eventCollector, words); - } - public void assertDoesNotContainEvent(String unexpectedEvent) { MoreAsserts.assertDoesNotContainEvent(eventCollector, unexpectedEvent); } diff --git a/src/test/java/com/google/devtools/build/lib/packages/PackageFactoryTest.java b/src/test/java/com/google/devtools/build/lib/packages/PackageFactoryTest.java index abf73f6a87f32f..fe6f4e151cde40 100644 --- a/src/test/java/com/google/devtools/build/lib/packages/PackageFactoryTest.java +++ b/src/test/java/com/google/devtools/build/lib/packages/PackageFactoryTest.java @@ -825,7 +825,7 @@ public void testPackageGroupNamedArguments() throws Exception { @Test public void testPackageSpecMinimal() throws Exception { Package pkg = expectEvalSuccess("package(default_visibility=[])"); - assertThat(pkg.getDefaultVisibility()).isNotNull(); + assertThat(pkg.getPackageArgs().defaultVisibility()).isNotNull(); } @Test @@ -854,14 +854,14 @@ public void testEmptyPackageSpecification() throws Exception { @Test public void testDefaultTestonly() throws Exception { Package pkg = expectEvalSuccess("package(default_testonly = 1)"); - assertThat(pkg.getDefaultTestOnly()).isTrue(); + assertThat(pkg.getPackageArgs().defaultTestOnly()).isTrue(); } @Test public void testDefaultDeprecation() throws Exception { String testMessage = "OMG PONIES!"; Package pkg = expectEvalSuccess("package(default_deprecation = \"" + testMessage + "\")"); - assertThat(pkg.getDefaultDeprecation()).isEqualTo(testMessage); + assertThat(pkg.getPackageArgs().defaultDeprecation()).isEqualTo(testMessage); } @Test @@ -930,7 +930,8 @@ public void testPackageFeatures() throws Exception { "package(features=['b', 'c'])", "sh_library(name='after')"); Package pkg = loadPackage("a"); - assertThat(pkg.getFeatures()).isEqualTo(FeatureSet.parse(ImmutableList.of("b", "c"))); + assertThat(pkg.getPackageArgs().features()) + .isEqualTo(FeatureSet.parse(ImmutableList.of("b", "c"))); } @Test @@ -1129,8 +1130,10 @@ public void testPackageDefaultEnvironments() throws Exception { " default_compatible_with=['//foo'],", " default_restricted_to=['//bar'],", ")"); - assertThat(pkg.getDefaultCompatibleWith()).containsExactly(Label.parseCanonical("//foo")); - assertThat(pkg.getDefaultRestrictedTo()).containsExactly(Label.parseCanonical("//bar")); + assertThat(pkg.getPackageArgs().defaultCompatibleWith()) + .containsExactly(Label.parseCanonical("//foo")); + assertThat(pkg.getPackageArgs().defaultRestrictedTo()) + .containsExactly(Label.parseCanonical("//bar")); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/pkgcache/PackageLoadingTest.java b/src/test/java/com/google/devtools/build/lib/pkgcache/PackageLoadingTest.java index 6b5d620d56b231..34fd1a27d8ea95 100644 --- a/src/test/java/com/google/devtools/build/lib/pkgcache/PackageLoadingTest.java +++ b/src/test/java/com/google/devtools/build/lib/pkgcache/PackageLoadingTest.java @@ -554,7 +554,7 @@ public void testPackageFeatures() throws Exception { "peach/BUILD", "package(features = ['crosstool_default_false'])", "cc_library(name = 'cc', srcs = ['cc.cc'])"); - assertThat(getPackage("peach").getFeatures()) + assertThat(getPackage("peach").getPackageArgs().features()) .isEqualTo(FeatureSet.parse(ImmutableList.of("crosstool_default_false"))); } diff --git a/src/test/java/com/google/devtools/build/lib/query2/cquery/BUILD b/src/test/java/com/google/devtools/build/lib/query2/cquery/BUILD index 54e9b61d842c83..095797ed6c9db7 100644 --- a/src/test/java/com/google/devtools/build/lib/query2/cquery/BUILD +++ b/src/test/java/com/google/devtools/build/lib/query2/cquery/BUILD @@ -137,14 +137,12 @@ java_test( "//src/main/java/com/google/devtools/build/lib/query2/engine", "//src/main/java/com/google/devtools/build/lib/query2/query/aspectresolvers", "//src/main/java/com/google/devtools/build/lib/util:filetype", - "//src/main/java/com/google/devtools/build/lib/util:pair", "//src/main/protobuf:analysis_v2_java_proto", "//src/main/protobuf:build_java_proto", "//src/test/java/com/google/devtools/build/lib/analysis/util", "//third_party:guava", "//third_party:junit4", "//third_party:truth", - "//third_party/protobuf:protobuf_java", ], ) diff --git a/src/test/java/com/google/devtools/build/lib/query2/cquery/ProtoOutputFormatterCallbackTest.java b/src/test/java/com/google/devtools/build/lib/query2/cquery/ProtoOutputFormatterCallbackTest.java index 6f11ee325d8e8d..5819474a4067bb 100644 --- a/src/test/java/com/google/devtools/build/lib/query2/cquery/ProtoOutputFormatterCallbackTest.java +++ b/src/test/java/com/google/devtools/build/lib/query2/cquery/ProtoOutputFormatterCallbackTest.java @@ -18,7 +18,6 @@ import static com.google.devtools.build.lib.packages.Attribute.attr; import static com.google.devtools.build.lib.packages.BuildType.LABEL_LIST; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.eventbus.EventBus; import com.google.devtools.build.lib.analysis.AnalysisProtosV2; @@ -41,13 +40,6 @@ import com.google.devtools.build.lib.query2.proto.proto2api.Build.ConfiguredRuleInput; import com.google.devtools.build.lib.query2.query.aspectresolvers.AspectResolver.Mode; import com.google.devtools.build.lib.util.FileTypeSet; -import com.google.devtools.build.lib.util.Pair; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.InputStream; import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; @@ -325,103 +317,31 @@ public void testAlias_withSelect() throws Exception { .containsAtLeast("//test:my_alias_rule", "//test:config1", "//test:target1"); } - @Test - public void testStreamedProtoAndProtoOutputsAreEquivalent() throws Exception { - MockRule depsRule = - () -> - MockRule.define( - "my_rule", - (builder, env) -> - builder.add(attr("deps", LABEL_LIST).allowedFileTypes(FileTypeSet.ANY_FILE))); - ConfiguredRuleClassProvider ruleClassProvider = setRuleClassProviders(depsRule).build(); - helper.useRuleClassProvider(ruleClassProvider); - - writeFile( - "test/BUILD", - "my_rule(name = 'my_rule',", - " deps = select({", - " ':garfield': ['lasagna.java', 'naps.java'],", - " '//conditions:default': ['mondays.java']", - " })", - ")", - "config_setting(", - " name = 'garfield',", - " values = {'foo': 'cat'}", - ")"); - getHelper().useConfiguration("--foo=cat"); - AnalysisProtosV2.CqueryResult protoOutput = getOutput("//test:my_rule", ruleClassProvider); - ImmutableList streamedProtoOutput = - getStreamedOutput("//test:my_rule", ruleClassProvider); - AnalysisProtosV2.CqueryResult.Builder combinedStreamedProtoBuilder = - AnalysisProtosV2.CqueryResult.newBuilder(); - for (AnalysisProtosV2.CqueryResult result : streamedProtoOutput) { - if (!result.getResultsList().isEmpty()) { - combinedStreamedProtoBuilder.addAllResults(result.getResultsList()); - } - if (!result.getConfigurationsList().isEmpty()) { - combinedStreamedProtoBuilder.addAllConfigurations(result.getConfigurationsList()); - } - } - assertThat(protoOutput) - .ignoringRepeatedFieldOrder() - .isEqualTo(combinedStreamedProtoBuilder.build()); - } - private MockRule getSimpleRule() { return () -> MockRule.define("simple_rule"); } private AnalysisProtosV2.CqueryResult getOutput( String queryExpression, RuleClassProvider ruleClassProvider) throws Exception { - CodedInputStream codedIn = - getInputStreamsWithData(queryExpression, ruleClassProvider, OutputType.BINARY).getSecond(); - return AnalysisProtosV2.CqueryResult.parser() - .parseFrom(codedIn, ExtensionRegistry.getEmptyRegistry()); - } - - private ImmutableList getStreamedOutput( - String queryExpression, RuleClassProvider ruleClassProvider) throws Exception { - InputStream in = - getInputStreamsWithData(queryExpression, ruleClassProvider, OutputType.DELIMITED_BINARY) - .getFirst(); - ImmutableList.Builder builder = new ImmutableList.Builder<>(); - AnalysisProtosV2.CqueryResult result; - while ((result = - AnalysisProtosV2.CqueryResult.parser() - .parseDelimitedFrom(in, ExtensionRegistry.getEmptyRegistry())) - != null) { - builder.add(result); - } - return builder.build(); - } - - private Pair getInputStreamsWithData( - String queryExpression, RuleClassProvider ruleClassProvider, OutputType outputType) - throws Exception { QueryExpression expression = QueryParser.parse(queryExpression, getDefaultFunctions()); Set targetPatternSet = new LinkedHashSet<>(); expression.collectTargetPatterns(targetPatternSet); helper.setQuerySettings(Setting.NO_IMPLICIT_DEPS); PostAnalysisQueryEnvironment env = ((ConfiguredTargetQueryHelper) helper).getPostAnalysisQueryEnvironment(targetPatternSet); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - CodedOutputStream codedOut = CodedOutputStream.newInstance(out); + ProtoOutputFormatterCallback callback = new ProtoOutputFormatterCallback( reporter, options, - out, - codedOut, + /* out= */ null, getHelper().getSkyframeExecutor(), env.getAccessor(), options.aspectDeps.createResolver( getHelper().getPackageManager(), NullEventHandler.INSTANCE), - outputType, + OutputType.BINARY, ruleClassProvider); env.evaluateQuery(expression, callback); - codedOut.flush(); - ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); - CodedInputStream codedIn = CodedInputStream.newInstance(in); - return new Pair<>(in, codedIn); + return callback.getProtoResult(); } } diff --git a/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java b/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java index 08d819a4fd384c..ce8918d669c4e4 100644 --- a/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java +++ b/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java @@ -29,6 +29,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; +import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsUtil; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; @@ -374,7 +375,7 @@ protected SkyframeExecutor createSkyframeExecutor(ConfiguredRuleClassProvider ru BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), + YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), @@ -416,7 +417,7 @@ protected SkyframeExecutor createSkyframeExecutor(ConfiguredRuleClassProvider ru CheckDirectDepsMode.WARNING)) .add( PrecomputedValue.injected( - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of())) + YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, ImmutableList.of())) .add( PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, diff --git a/src/test/java/com/google/devtools/build/lib/remote/BUILD b/src/test/java/com/google/devtools/build/lib/remote/BUILD index 2f3da6d42a7562..58f0345d83c454 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/BUILD +++ b/src/test/java/com/google/devtools/build/lib/remote/BUILD @@ -169,6 +169,7 @@ java_test( "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper:credential_module", "//src/main/java/com/google/devtools/build/lib/dynamic", "//src/main/java/com/google/devtools/build/lib/remote", + "//src/main/java/com/google/devtools/build/lib/remote/util", "//src/main/java/com/google/devtools/build/lib/standalone", "//src/main/java/com/google/devtools/build/lib/util:os", "//src/main/java/com/google/devtools/build/lib/vfs", diff --git a/src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTest.java b/src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTest.java index ffecdfb109b082..412bdd0bc90e21 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTest.java @@ -22,10 +22,12 @@ import static org.junit.Assume.assumeFalse; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Sets; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.BuildFailedException; import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialModule; import com.google.devtools.build.lib.dynamic.DynamicExecutionModule; +import com.google.devtools.build.lib.remote.util.DigestUtil; import com.google.devtools.build.lib.remote.util.IntegrationTestUtils.WorkerInstance; import com.google.devtools.build.lib.runtime.BlazeModule; import com.google.devtools.build.lib.runtime.BlazeRuntime; @@ -796,4 +798,61 @@ public void downloadToplevel_symlinkTree() throws Exception { assertValidOutputFile("foo-link/file-2", "2"); assertValidOutputFile("foo-link/file-3", "3"); } + + @Test + public void leaseExtension() throws Exception { + // Test that Bazel will extend the leases for remote output by sending FindMissingBlobs calls + // periodically to remote server. The test assumes remote server will set mtime of referenced + // blobs to `now`. + write( + "BUILD", + "genrule(", + " name = 'foo',", + " srcs = [],", + " outs = ['out/foo.txt'],", + " cmd = 'echo -n foo > $@',", + ")", + "genrule(", + " name = 'foobar',", + " srcs = [':foo'],", + " outs = ['out/foobar.txt'],", + // We need the action lasts more than --experimental_remote_cache_ttl so Bazel has the + // chance to extend the lease + " cmd = 'sleep 2 && cat $(location :foo) > $@ && echo bar >> $@',", + ")"); + addOptions("--experimental_remote_cache_ttl=1s", "--experimental_remote_cache_lease_extension"); + var content = "foo".getBytes(UTF_8); + var hashCode = getFileSystem().getDigestFunction().getHashFunction().hashBytes(content); + var digest = DigestUtil.buildDigest(hashCode.asBytes(), content.length).getHash(); + // Calculate the blob path in CAS. This is specific to the remote worker. See + // {@link DiskCacheClient#getPath()}. + var blobPath = + getFileSystem() + .getPath(worker.getCasPath()) + .getChild("cas") + .getChild(digest.substring(0, 2)) + .getChild(digest); + var mtimes = Sets.newConcurrentHashSet(); + // Observe the mtime of the blob in background. + var thread = + new Thread( + () -> { + while (!Thread.currentThread().isInterrupted()) { + try { + mtimes.add(blobPath.getLastModifiedTime()); + } catch (IOException ignored) { + // Intentionally ignored + } + } + }); + thread.start(); + + buildTarget("//:foobar"); + waitDownloads(); + + thread.interrupt(); + thread.join(); + // We should be able to observe more than 1 mtime if the server extends the lease. + assertThat(mtimes.size()).isGreaterThan(1); + } } diff --git a/src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTestBase.java b/src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTestBase.java index 7039d757d14bd7..8f6c0b3ad77610 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTestBase.java +++ b/src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTestBase.java @@ -1395,6 +1395,36 @@ public void remoteTreeFilesExpiredBetweenBuilds_rerunGeneratingActions() throws assertValidOutputFile("a/bar.out", "file-inside\nupdated bar" + lineSeparator()); } + @Test + public void nonDeclaredSymlinksFromLocalActions() throws Exception { + write( + "BUILD", + "genrule(", + " name = 'foo',", + " srcs = [],", + " outs = ['foo.txt'],", + " cmd = 'echo foo > $@',", + ")", + "genrule(", + " name = 'foo-link',", + " srcs = [':foo'],", + " outs = ['foo.link'],", + " cmd = 'ln -s foo.txt $@',", + " local = True,", + ")", + "genrule(", + " name = 'foobar',", + " srcs = [':foo-link'],", + " outs = ['foobar.txt'],", + " cmd = 'cat $(location :foo-link) > $@ && echo bar >> $@',", + " local = True,", + ")"); + + buildTarget("//:foobar"); + + assertValidOutputFile("foobar.txt", "foo\nbar\n"); + } + protected void assertOutputsDoNotExist(String target) throws Exception { for (Artifact output : getArtifacts(target)) { assertWithMessage( diff --git a/src/test/java/com/google/devtools/build/lib/remote/ByteStreamUploaderTest.java b/src/test/java/com/google/devtools/build/lib/remote/ByteStreamUploaderTest.java index fd19dddc9e9114..53d433faa03e21 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/ByteStreamUploaderTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/ByteStreamUploaderTest.java @@ -21,6 +21,7 @@ import static org.mockito.ArgumentMatchers.any; import build.bazel.remote.execution.v2.Digest; +import build.bazel.remote.execution.v2.DigestFunction; import build.bazel.remote.execution.v2.RequestMetadata; import com.github.luben.zstd.Zstd; import com.github.luben.zstd.ZstdInputStream; @@ -165,7 +166,8 @@ public void singleBlobUploadShouldWork() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE * 2 + 1]; new Random().nextBytes(blob); @@ -192,7 +194,8 @@ public void singleChunkCompressedUploadAlreadyExists() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /* maximumOpenFiles= */ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = {'A'}; @@ -232,8 +235,7 @@ public void onError(Throwable throwable) { } @Override - public void onCompleted() { - } + public void onCompleted() {} }; } }); @@ -256,7 +258,8 @@ public void progressiveUploadShouldWork() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, 3, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE * 2 + 1]; new Random().nextBytes(blob); @@ -372,7 +375,8 @@ public void progressiveCompressedUploadShouldWork() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, 300, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); int chunkSize = 1024; int skipSize = chunkSize + 1; @@ -491,7 +495,8 @@ public void progressiveCompressedUploadSeesAlreadyExistsAtTheEnd() throws Except CallCredentialsProvider.NO_CREDENTIALS, 300, retrier, - /* maximumOpenFiles= */ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); int chunkSize = 1024; byte[] blob = new byte[chunkSize * 2 + 1]; @@ -549,7 +554,8 @@ public void concurrentlyCompletedUploadIsNotRetried() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, 1, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE * 2 + 1]; new Random().nextBytes(blob); @@ -607,7 +613,8 @@ public void unimplementedQueryShouldRestartUpload() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, 3, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE * 2 + 1]; new Random().nextBytes(blob); @@ -676,7 +683,8 @@ public void earlyWriteResponseShouldCompleteUpload() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, 3, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE * 2 + 1]; new Random().nextBytes(blob); @@ -713,7 +721,8 @@ public void incorrectCommittedSizeFailsCompletedUpload() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, 3, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE * 2 + 1]; new Random().nextBytes(blob); @@ -766,7 +775,8 @@ public void incorrectCommittedSizeDoesNotFailIncompleteUpload() throws Exception CallCredentialsProvider.NO_CREDENTIALS, 300, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE * 2 + 1]; new Random().nextBytes(blob); @@ -798,7 +808,8 @@ public void multipleBlobsUploadShouldWork() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); int numUploads = 10; Map blobsByHash = Maps.newHashMap(); @@ -830,7 +841,8 @@ public void tooManyFilesIOException_adviseMaximumOpenFilesFlag() throws Exceptio CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE]; Chunker chunker = Mockito.mock(Chunker.class); Digest digest = DIGEST_UTIL.compute(blob); @@ -862,7 +874,8 @@ public void availablePermitsOpenFileSemaphore_fewerPermitsThanUploads_endWithAll CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - maximumOpenFiles); + maximumOpenFiles, + /* digestFunction= */ DigestFunction.Value.SHA256); assertThat(uploader.getOpenedFilePermits().availablePermits()).isEqualTo(999); @@ -900,7 +913,8 @@ public void noMaximumOpenFilesFlags_nullSemaphore() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); assertThat(uploader.getOpenedFilePermits()).isNull(); int numUploads = 10; @@ -936,7 +950,8 @@ public void contextShouldBePreservedUponRetries() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); List toUpload = ImmutableList.of("aaaaaaaaaa", "bbbbbbbbbb", "cccccccccc"); Map chunkers = Maps.newHashMapWithExpectedSize(toUpload.size()); @@ -1066,7 +1081,8 @@ public int maxConcurrency() { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE]; Chunker chunker = Chunker.builder().setInput(blob).setChunkSize(CHUNK_SIZE).build(); @@ -1127,7 +1143,8 @@ public void errorsShouldBeReported() throws IOException, InterruptedException { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE]; Chunker chunker = Chunker.builder().setInput(blob).setChunkSize(CHUNK_SIZE).build(); @@ -1163,7 +1180,8 @@ public void failureInRetryExecutorShouldBeHandled() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); serviceRegistry.addService( new ByteStreamImplBase() { @@ -1202,7 +1220,8 @@ public void resourceNameWithoutInstanceName() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); serviceRegistry.addService( new ByteStreamImplBase() { @@ -1234,6 +1253,50 @@ public void onCompleted() { uploader.uploadBlob(context, digest, chunker); } + @Test + public void resourceWithNewStyleDigestFunction() throws Exception { + RemoteRetrier retrier = + TestUtils.newRemoteRetrier(() -> mockBackoff, (e) -> true, retryService); + ByteStreamUploader uploader = + new ByteStreamUploader( + /* instanceName= */ null, + referenceCountedChannel, + CallCredentialsProvider.NO_CREDENTIALS, + /* callTimeoutSecs= */ 60, + retrier, + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.BLAKE3); + + serviceRegistry.addService( + new ByteStreamImplBase() { + @Override + public StreamObserver write(StreamObserver response) { + return new StreamObserver() { + @Override + public void onNext(WriteRequest writeRequest) { + // Test that the resource name contains the digest function. + assertThat(writeRequest.getResourceName()).contains("blobs/blake3/"); + } + + @Override + public void onError(Throwable throwable) {} + + @Override + public void onCompleted() { + response.onNext(WriteResponse.newBuilder().setCommittedSize(1).build()); + response.onCompleted(); + } + }; + } + }); + + byte[] blob = new byte[1]; + Chunker chunker = Chunker.builder().setInput(blob).setChunkSize(CHUNK_SIZE).build(); + Digest digest = DIGEST_UTIL.compute(blob); + + uploader.uploadBlob(context, digest, chunker); + } + @Test public void nonRetryableStatusShouldNotBeRetried() throws Exception { RemoteRetrier retrier = @@ -1246,7 +1309,8 @@ public void nonRetryableStatusShouldNotBeRetried() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); AtomicInteger numCalls = new AtomicInteger(); @@ -1299,7 +1363,8 @@ public void refresh() throws IOException { callCredentialsProvider, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE * 2 + 1]; new Random().nextBytes(blob); @@ -1355,7 +1420,8 @@ public void refresh() throws IOException { callCredentialsProvider, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE * 2 + 1]; new Random().nextBytes(blob); @@ -1425,7 +1491,8 @@ public void failureAfterUploadCompletes() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - -1); + -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE - 1]; new Random().nextBytes(blob); @@ -1484,7 +1551,8 @@ public void testCompressedUploads() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE * 2 + 1]; new Random().nextBytes(blob); diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java index 26e374d05a958c..df3087daf62c1b 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java @@ -21,7 +21,7 @@ import static org.mockito.AdditionalAnswers.answerVoid; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.spy; import build.bazel.remote.execution.v2.Action; import build.bazel.remote.execution.v2.ActionCacheGrpc.ActionCacheImplBase; @@ -811,10 +811,9 @@ public void updateActionResult( } } }); - ByteStreamImplBase mockByteStreamImpl = Mockito.mock(ByteStreamImplBase.class); + ByteStreamImplBase mockByteStreamImpl = spy(ByteStreamImplBase.class); serviceRegistry.addService(mockByteStreamImpl); - when(mockByteStreamImpl.write(ArgumentMatchers.>any())) - .thenAnswer( + doAnswer( new Answer>() { private int numErrors = 4; @@ -865,7 +864,9 @@ public void onError(Throwable t) { } }; } - }); + }) + .when(mockByteStreamImpl) + .write(any()); doAnswer( answerVoid( (QueryWriteStatusRequest request, diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java index c03e0a6d7395f9..faa020f21e6311 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java @@ -22,6 +22,7 @@ import static org.mockito.AdditionalAnswers.answerVoid; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -31,6 +32,7 @@ import build.bazel.remote.execution.v2.Command; import build.bazel.remote.execution.v2.ContentAddressableStorageGrpc.ContentAddressableStorageImplBase; import build.bazel.remote.execution.v2.Digest; +import build.bazel.remote.execution.v2.DigestFunction; import build.bazel.remote.execution.v2.Directory; import build.bazel.remote.execution.v2.ExecuteRequest; import build.bazel.remote.execution.v2.ExecuteResponse; @@ -729,9 +731,8 @@ public void findMissingBlobs( }; serviceRegistry.addService(ServerInterceptors.intercept(cas, new RequestHeadersValidator())); - ByteStreamImplBase mockByteStreamImpl = Mockito.mock(ByteStreamImplBase.class); - when(mockByteStreamImpl.write(ArgumentMatchers.>any())) - .thenAnswer(blobWriteAnswer("xyz".getBytes(UTF_8))); + ByteStreamImplBase mockByteStreamImpl = spy(ByteStreamImplBase.class); + doAnswer(blobWriteAnswer("xyz".getBytes(UTF_8))).when(mockByteStreamImpl).write(any()); serviceRegistry.addService( ServerInterceptors.intercept(mockByteStreamImpl, new RequestHeadersValidator())); @@ -807,7 +808,7 @@ public void getActionResult( .setResponse(Any.pack(ExecuteResponse.newBuilder().setResult(actionResult).build())) .build(); - ExecutionImplBase mockExecutionImpl = Mockito.mock(ExecutionImplBase.class); + ExecutionImplBase mockExecutionImpl = spy(ExecutionImplBase.class); // Flow of this test: // - call execute, get retriable gRPC error // - retry: call execute, get retriable Operation error @@ -815,7 +816,7 @@ public void getActionResult( // - retry: call waitExecute, get a retriable gRPC error // - retry: call waitExecute, get retriable Operation error // - retry: call execute, get successful operation, ignore further errors. - Mockito.doAnswer(answerWith(null, Status.UNAVAILABLE)) + doAnswer(answerWith(null, Status.UNAVAILABLE)) .doAnswer(answerWith(operationWithExecuteError, Status.OK)) .doAnswer(answerWith(unfinishedOperation, Status.UNAVAILABLE)) .doAnswer(answerWith(opSuccess, Status.UNAVAILABLE)) // last status should be ignored. @@ -823,7 +824,7 @@ public void getActionResult( .execute( ArgumentMatchers.any(), ArgumentMatchers.>any()); - Mockito.doAnswer(answerWith(null, Status.UNAVAILABLE)) + doAnswer(answerWith(null, Status.UNAVAILABLE)) .doAnswer(answerWith(operationWithExecuteError, Status.OK)) .when(mockExecutionImpl) .waitExecution( @@ -853,11 +854,12 @@ public void findMissingBlobs( } }); - ByteStreamImplBase mockByteStreamImpl = Mockito.mock(ByteStreamImplBase.class); - when(mockByteStreamImpl.write(ArgumentMatchers.>any())) - .thenAnswer(blobWriteAnswerError()) // Error on the input file. - .thenAnswer(blobWriteAnswerError()) // Error on the input file again. - .thenAnswer(blobWriteAnswer("xyz".getBytes(UTF_8))); // Upload input file successfully. + ByteStreamImplBase mockByteStreamImpl = spy(ByteStreamImplBase.class); + doAnswer(blobWriteAnswerError()) // Error on the input file. + .doAnswer(blobWriteAnswerError()) // Error on the input file again. + .doAnswer(blobWriteAnswer("xyz".getBytes(UTF_8))) // Upload input file successfully. + .when(mockByteStreamImpl) + .write(any()); doAnswer( answerVoid( (QueryWriteStatusRequest request, @@ -871,7 +873,7 @@ public void findMissingBlobs( })) .when(mockByteStreamImpl) .queryWriteStatus(any(), any()); - Mockito.doAnswer( + doAnswer( invocationOnMock -> { @SuppressWarnings("unchecked") StreamObserver responseObserver = @@ -951,26 +953,22 @@ public void getActionResult( .setResponse(Any.pack(ExecuteResponse.newBuilder().setResult(actionResult).build())) .build(); - ExecutionImplBase mockExecutionImpl = Mockito.mock(ExecutionImplBase.class); + ExecutionImplBase mockExecutionImpl = spy(ExecutionImplBase.class); // Flow of this test: // - call execute, get an Operation, then a retriable gRPC error // - retry: call waitExecute, get NOT_FOUND (operation lost) // - retry: call execute, get NOT_FOUND (operation lost) // - retry: call execute, get an Operation, then a retriable gRPC error // - retry: call waitExecute, get successful operation, ignore further errors. - Mockito.doAnswer(answerWith(unfinishedOperation, Status.UNAVAILABLE)) + doAnswer(answerWith(unfinishedOperation, Status.UNAVAILABLE)) .doAnswer(answerWith(unfinishedOperation, Status.NOT_FOUND)) .doAnswer(answerWith(unfinishedOperation, Status.UNAVAILABLE)) .when(mockExecutionImpl) - .execute( - ArgumentMatchers.any(), - ArgumentMatchers.>any()); - Mockito.doAnswer(answerWith(unfinishedOperation, Status.NOT_FOUND)) + .execute(any(), any()); + doAnswer(answerWith(unfinishedOperation, Status.NOT_FOUND)) .doAnswer(answerWith(opSuccess, Status.UNAVAILABLE)) // This error is ignored. .when(mockExecutionImpl) - .waitExecution( - ArgumentMatchers.any(), - ArgumentMatchers.>any()); + .waitExecution(any(), any()); serviceRegistry.addService(mockExecutionImpl); serviceRegistry.addService( @@ -989,10 +987,9 @@ public void findMissingBlobs( } }); - ByteStreamImplBase mockByteStreamImpl = Mockito.mock(ByteStreamImplBase.class); - when(mockByteStreamImpl.write(ArgumentMatchers.>any())) - .thenAnswer(blobWriteAnswer("xyz".getBytes(UTF_8))); // Upload input file successfully. - Mockito.doAnswer( + ByteStreamImplBase mockByteStreamImpl = spy(ByteStreamImplBase.class); + doAnswer(blobWriteAnswer("xyz".getBytes(UTF_8))).when(mockByteStreamImpl).write(any()); + doAnswer( invocationOnMock -> { @SuppressWarnings("unchecked") StreamObserver responseObserver = @@ -1110,7 +1107,8 @@ public void findMissingBlobs( } }); String stdOutResourceName = - getResourceName(remoteOptions.remoteInstanceName, stdOutDigest, false); + getResourceName( + remoteOptions.remoteInstanceName, stdOutDigest, false, DigestFunction.Value.SHA256); serviceRegistry.addService( new ByteStreamImplBase() { @Override @@ -1171,7 +1169,8 @@ public void findMissingBlobs( } }); String stdOutResourceName = - getResourceName(remoteOptions.remoteInstanceName, stdOutDigest, false); + getResourceName( + remoteOptions.remoteInstanceName, stdOutDigest, false, DigestFunction.Value.SHA256); serviceRegistry.addService( new ByteStreamImplBase() { @Override @@ -1297,7 +1296,10 @@ public void getActionResult( }); String dummyTreeResourceName = getResourceName( - remoteOptions.remoteInstanceName, DUMMY_OUTPUT_DIRECTORY.getTreeDigest(), false); + remoteOptions.remoteInstanceName, + DUMMY_OUTPUT_DIRECTORY.getTreeDigest(), + false, + DigestFunction.Value.SHA256); serviceRegistry.addService( new ByteStreamImplBase() { private boolean first = true; @@ -1508,17 +1510,17 @@ public void read(ReadRequest request, StreamObserver responseObser .build(); final WaitExecutionRequest waitExecutionRequest = WaitExecutionRequest.newBuilder().setName(opName).build(); - ExecutionImplBase mockExecutionImpl = Mockito.mock(ExecutionImplBase.class); + ExecutionImplBase mockExecutionImpl = spy(ExecutionImplBase.class); // Flow of this test: // - call execute, get an unfinished Operation, then the stream completes // - call waitExecute, get an unfinished Operation, then the stream completes // - call waitExecute, get a finished Operation - Mockito.doAnswer(answerWith(unfinishedOperation, Status.OK)) + doAnswer(answerWith(unfinishedOperation, Status.OK)) .when(mockExecutionImpl) .execute( ArgumentMatchers.any(), ArgumentMatchers.>any()); - Mockito.doAnswer(answerWith(unfinishedOperation, Status.OK)) + doAnswer(answerWith(unfinishedOperation, Status.OK)) .doAnswer(answerWith(completeOperation, Status.OK)) .when(mockExecutionImpl) .waitExecution( diff --git a/src/test/java/com/google/devtools/build/lib/remote/util/IntegrationTestUtils.java b/src/test/java/com/google/devtools/build/lib/remote/util/IntegrationTestUtils.java index 0eca00a81e273c..7074f41e20750c 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/util/IntegrationTestUtils.java +++ b/src/test/java/com/google/devtools/build/lib/remote/util/IntegrationTestUtils.java @@ -257,5 +257,9 @@ private static void deleteDir(PathFragment path) throws IOException { public int getPort() { return port; } + + public PathFragment getCasPath() { + return casPath; + } } } diff --git a/src/test/java/com/google/devtools/build/lib/rules/LabelBuildSettingTest.java b/src/test/java/com/google/devtools/build/lib/rules/LabelBuildSettingTest.java index 0d88e1b9f9d35b..c43e1b88c18be5 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/LabelBuildSettingTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/LabelBuildSettingTest.java @@ -25,6 +25,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; +import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsUtil; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; @@ -54,8 +55,7 @@ protected ImmutableList extraPrecomputedValues() { ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl())), PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false), PrecomputedValue.injected(ModuleFileFunction.MODULE_OVERRIDES, ImmutableMap.of()), - PrecomputedValue.injected( - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), + PrecomputedValue.injected(YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), PrecomputedValue.injected( BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( diff --git a/src/test/java/com/google/devtools/build/lib/rules/cpp/BUILD b/src/test/java/com/google/devtools/build/lib/rules/cpp/BUILD index 5bf38f2ea3ed7c..7fca39b50fae6f 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/cpp/BUILD +++ b/src/test/java/com/google/devtools/build/lib/rules/cpp/BUILD @@ -110,6 +110,28 @@ java_test( ], ) +java_test( + name = "CcBinaryThinLtoObjDirTest", + srcs = ["CcBinaryThinLtoObjDirTest.java"], + deps = [ + "//src/main/java/com/google/devtools/build/lib/actions", + "//src/main/java/com/google/devtools/build/lib/actions:artifacts", + "//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster", + "//src/main/java/com/google/devtools/build/lib/analysis:configured_target", + "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/lib/rules/cpp", + "//src/main/java/com/google/devtools/build/lib/skyframe:configured_target_key", + "//src/main/java/com/google/devtools/build/lib/skyframe:rule_configured_target_value", + "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", + "//src/test/java/com/google/devtools/build/lib/actions/util", + "//src/test/java/com/google/devtools/build/lib/analysis/util", + "//src/test/java/com/google/devtools/build/lib/packages:testutil", + "//third_party:guava", + "//third_party:junit4", + "//third_party:truth", + ], +) + java_test( name = "CcBinarySplitFunctionsTest", srcs = ["CcBinarySplitFunctionsTest.java"], diff --git a/src/test/java/com/google/devtools/build/lib/rules/cpp/CcBinaryThinLtoObjDirTest.java b/src/test/java/com/google/devtools/build/lib/rules/cpp/CcBinaryThinLtoObjDirTest.java new file mode 100644 index 00000000000000..048cb049ac1bc6 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/rules/cpp/CcBinaryThinLtoObjDirTest.java @@ -0,0 +1,2100 @@ +// Copyright 2020 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.truth.Truth.assertThat; +import static java.util.Arrays.stream; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionAnalysisMetadata; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.util.ActionsTestUtil; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.analysis.util.AnalysisMock; +import com.google.devtools.build.lib.analysis.util.BuildViewTestCase; +import com.google.devtools.build.lib.cmdline.RepositoryName; +import com.google.devtools.build.lib.packages.util.Crosstool.CcToolchainConfig; +import com.google.devtools.build.lib.packages.util.MockCcSupport; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.Tool; +import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey; +import com.google.devtools.build.lib.skyframe.RuleConfiguredTargetValue; +import com.google.devtools.build.lib.vfs.PathFragment; +import java.io.IOException; +import java.util.List; +import java.util.stream.Stream; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for cc_binary with treeArtifacts, ThinLTO and separate obj dir for thinlto. */ +@RunWith(JUnit4.class) +public class CcBinaryThinLtoObjDirTest extends BuildViewTestCase { + + private String targetName = "bin"; + + private ConfiguredTarget getCurrentTarget() throws Exception { + return getConfiguredTarget("//pkg:" + targetName); + } + + private CppLinkAction getLinkAction() throws Exception { + ConfiguredTarget pkg = getCurrentTarget(); + Artifact pkgArtifact = getFilesToBuild(pkg).getSingleton(); + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(pkgArtifact); + assertThat(linkAction.getOutputs()).containsExactly(pkgArtifact); + return linkAction; + } + + private LtoBackendAction getBackendAction(String path) throws Exception { + return (LtoBackendAction) getPredecessorByInputName(getLinkAction(), path); + } + + private String getRootExecPath() throws Exception { + ConfiguredTarget pkg = getCurrentTarget(); + Artifact pkgArtifact = getFilesToBuild(pkg).getSingleton(); + return pkgArtifact.getRoot().getExecPathString(); + } + + private CppLinkAction getIndexAction(LtoBackendAction backendAction) throws Exception { + return (CppLinkAction) + getPredecessorByInputName( + backendAction, + (backendAction.getPrimaryOutput().getExecPathString() + ".thinlto.bc") + .replaceFirst(".lto-obj/", ".lto/")); + } + + @Before + public void createBasePkg() throws IOException { + scratch.overwriteFile( + "base/BUILD", "cc_library(name = 'system_malloc', visibility = ['//visibility:public'])"); + } + + public void createBuildFiles(String... extraCcBinaryParameters) throws Exception { + scratch.file( + "pkg/BUILD", + "load(':do_gen.bzl', 'test_generation', 'test_generation_2', 'test_generation_empty')", + "package(features = ['thin_lto', 'use_lto_native_object_directory'])", + "", + "test_generation(", + " name = 'tree',", + ")", + "test_generation_2(", + " name = 'tree_2',", + ")", + "test_generation_empty(", + " name = 'tree_empty',", + ")", + "cc_binary(name = '" + targetName + "',", + " srcs = ['binfile.cc', ],", + " deps = [ ':lib', ':tree', ':tree_2', 'tree_empty'], ", + String.join("", extraCcBinaryParameters), + " malloc = '//base:system_malloc')", + "cc_library(name = 'lib',", + " srcs = ['libfile.cc'],", + " hdrs = ['libfile.h'],", + " linkstamp = 'linkstamp.cc',", + " )"); + scratch.file( + "pkg/do_gen.bzl", + "def _create_cc_impl(ctx):", + " directory = ctx.actions.declare_directory(ctx.label.name + \"_gen_cc\")", + " ctx.actions.run_shell(", + " command = \"echo -e '#include \\\"pkg/treelib.h\\\"\\n" + + "Foo::~Foo() { }' > %s/file1.cc\" % directory.path,", + " outputs=[directory]", + " )", + " return DefaultInfo(files=depset([directory]))", + "", + "_create_cc = rule(implementation=_create_cc_impl)", + "def test_generation(name):", + " _create_cc(name=name + \"_ccgen\")", + "", + " native.cc_library(", + " name = name,", + " hdrs = [\"treelib.h\",],", + " srcs = [\":\" + name + \"_ccgen\",]", + ")", + "", + "def _create_cc_impl_2(ctx):", + " directory = ctx.actions.declare_directory(ctx.label.name + \"_gen_cc_2\")", + " ctx.actions.run_shell(", + " command = \"echo -e '#include \\\"pkg/treelib_2.h\\\"\\n" + + "int two() { return 2; }' > %s/file1.cc\" % directory.path +" + + " \"echo -e '#include \\\"pkg/treelib_2.h\\\"\\n" + + "int three() { return 3; }' > %s/file2.cc\" % directory.path,", + " outputs=[directory]", + " )", + " return DefaultInfo(files=depset([directory]))", + "", + "_create_cc_2 = rule(implementation=_create_cc_impl_2)", + "def test_generation_2(name):", + " _create_cc_2(name=name + \"_ccgen_2\")", + "", + " native.cc_library(", + " name = name,", + " hdrs = [\"treelib_2.h\",],", + " srcs = [\":\" + name + \"_ccgen_2\",]", + ")", + "", + "def _create_cc_impl_empty(ctx):", + " directory = ctx.actions.declare_directory(ctx.label.name + \"_gen_cc_empty\")", + " ctx.actions.run_shell(", + " command = \"echo 'empty'\",", + " outputs=[directory]", + " )", + " return DefaultInfo(files=depset([directory]))", + "", + "_create_cc_empty = rule(implementation=_create_cc_impl_empty)", + "def test_generation_empty(name):", + " _create_cc_empty(name=name + \"_ccgen_empty\")", + "", + " native.cc_library(", + " name = name,", + " srcs = [\":\" + name + \"_ccgen_empty\",]", + ")"); + + scratch.file("pkg/treelib.h", "class Foo{ public: ~Foo(); };"); + scratch.file("pkg/treelib_2.h", "int two(); int three();"); + + scratch.file( + "pkg/binfile.cc", + "#include \"pkg/libfile.h\"", + "#include \"pkg/treelib.h\"", + "#include \"pkg/treelib_2.h\"", + "int main() {", + " Foo foo;", + " return pkg() + two() + three(); }"); + scratch.file("pkg/libfile.cc", "int pkg() { return 42; }"); + scratch.file("pkg/libfile.h", "int pkg();"); + scratch.file("pkg/linkstamp.cc"); + } + + public void createTestFiles(String extraTestParameters, String extraLibraryParameters) + throws Exception { + scratch.file( + "pkg/BUILD", + "load(':do_gen.bzl', 'test_generation')", + "package(features = ['thin_lto', 'use_lto_native_object_directory'])", + "", + "test_generation(", + " name = 'tree',", + ")", + "cc_test(", + " name = 'bin_test',", + " srcs = ['bin_test.cc', ],", + " deps = [ ':lib', ':tree', ], ", + extraTestParameters, + " malloc = '//base:system_malloc'", + ")", + "cc_test(", + " name = 'bin_test2',", + " srcs = ['bin_test2.cc', ],", + " deps = [ ':lib', ':tree', ], ", + extraTestParameters, + " malloc = '//base:system_malloc'", + ")", + "cc_library(", + " name = 'lib',", + " srcs = ['libfile.cc'],", + " hdrs = ['libfile.h'],", + extraLibraryParameters, + " linkstamp = 'linkstamp.cc',", + ")"); + scratch.file( + "pkg/do_gen.bzl", + "def _create_cc_impl(ctx):", + " directory = ctx.actions.declare_directory(ctx.label.name + \"_gen_cc\")", + " ctx.actions.run_shell(", + " command = \"echo -e '#include \\\"pkg/treelib.h\\\"\\n" + + "Foo::~Foo() { }' > %s/file.cc\" % directory.path,", + " outputs=[directory]", + " )", + " return DefaultInfo(files=depset([directory]))", + "", + "_create_cc = rule(implementation=_create_cc_impl)", + "def test_generation(name):", + " _create_cc(name=name + \"_ccgen\")", + "", + " native.cc_library(", + " name = name,", + " hdrs = [\"treelib.h\",],", + " srcs = [\":\" + name + \"_ccgen\",]", + ")"); + scratch.file("pkg/treelib.h", "class Foo{ public: ~Foo(); };"); + scratch.file( + "pkg/bin_test.cc", + "#include \"pkg/libfile.h\"", + "#include \"pkg/treelib.h\"", + "int main() { Foo foo; return pkg(); }"); + scratch.file( + "pkg/bin_test2.cc", + "#include \"pkg/libfile.h\"", + "#include \"pkg/treelib.h\"", + "int main() { Foo foo; return pkg(); }"); + scratch.file("pkg/libfile.cc", "int pkg() { return 42; }"); + scratch.file("pkg/libfile.h", "int pkg();"); + scratch.file("pkg/linkstamp.cc"); + } + + @Test + public void testActionGraph() throws Exception { + createBuildFiles(); + setupThinLTOCrosstool(CppRuleClasses.SUPPORTS_PIC); + useConfiguration("--noincompatible_make_thinlto_command_lines_standalone"); + + /* + We follow the chain from the final product backwards. + + binary <=[Link]= + .lto-obj/...o <=[LTOBackend]= + {.o.thinlto.bc,.o.imports} <=[LTOIndexing]= + .o <= [CppCompile] .cc + */ + ConfiguredTarget pkg = getCurrentTarget(); + CppLinkAction linkAction = getLinkAction(); + String rootExecPath = getRootExecPath(); + + assertThat(ActionsTestUtil.getFirstArtifactEndingWith(linkAction.getInputs(), "linkstamp.o")) + .isNotNull(); + + List commandLine = linkAction.getLinkCommandLineForTesting().getRawLinkArgv(); + String prefix = getTargetConfiguration().getOutputDirectory(RepositoryName.MAIN) + .getExecPathString(); + assertThat(commandLine) + .containsAtLeast( + prefix + "/bin/pkg/bin.lto.merged.o", + "thinlto_param_file=" + prefix + "/bin/pkg/bin-lto-final.params") + .inOrder(); + + // We have no bitcode files: all files have pkg/bin.lto/ + for (String arg : commandLine) { + if (arg.contains("_objs") && !arg.contains("linkstamp.o")) { + assertThat(arg).contains("pkg/bin.lto"); + } + } + + assertThat(artifactsToStrings(linkAction.getInputs())) + .containsAtLeast( + "bin pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o", + "bin pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/lib/libfile.pic.o", + "bin pkg/bin-2.params", + "bin pkg/bin-lto-final.params"); + + LtoBackendAction backendAction = + getBackendAction("pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o"); + assertThat(backendAction.getMnemonic()).isEqualTo("CcLtoBackendCompile"); + + assertThat(artifactsToStrings(backendAction.getInputs())) + .containsAtLeast( + "bin pkg/bin.lto/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o.thinlto.bc", + "bin pkg/bin.lto/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o.imports"); + + assertThat(backendAction.getArguments()) + .containsAtLeast( + "thinlto_index=" + + prefix + + "/bin/pkg/bin.lto/" + + rootExecPath + + "/pkg/_objs/bin/binfile.pic.o.thinlto.bc", + "thinlto_output_object_file=" + + prefix + + "/bin/pkg/bin.lto-obj/" + + rootExecPath + + "/pkg/_objs/bin/binfile.pic.o", + "thinlto_input_bitcode_file=" + prefix + "/bin/pkg/_objs/bin/binfile.pic.o"); + + CppLinkAction indexAction = getIndexAction(backendAction); + + RuleConfiguredTargetValue configuredTargetValue = + (RuleConfiguredTargetValue) + getSkyframeExecutor() + .getEvaluator() + .getExistingEntryAtCurrentlyEvaluatingVersion( + ConfiguredTargetKey.builder() + .setLabel(pkg.getLabel()) + .setConfiguration(getConfiguration(pkg)) + .build() + .toKey()) + .getValue(); + ImmutableList linkstampCompileActions = + configuredTargetValue.getActions().stream() + .filter(a -> a.getMnemonic().equals("CppLinkstampCompile")) + .collect(toImmutableList()); + assertThat(linkstampCompileActions).hasSize(1); + ActionAnalysisMetadata linkstampCompileAction = linkstampCompileActions.get(0); + assertThat(indexAction.getInputs().toList()) + .containsNoneIn(linkstampCompileAction.getOutputs()); + + assertThat(indexAction.getArguments()) + .containsAtLeast( + "param_file=" + prefix + "/bin/pkg/bin-lto-final.params", + "prefix_replace=" + + ";" + + prefix + + "/bin/pkg/bin.lto/" + + ";" + + prefix + + "/bin/pkg/bin.lto-obj/", + "thinlto_merged_object_file=" + prefix + "/bin/pkg/bin.lto.merged.o", + "object_suffix_replace=.indexing.o;.o"); + assertThat(indexAction.getArguments()) + .doesNotContain("thinlto_param_file=" + prefix + "/bin/pkg/bin-lto-final.params"); + + assertThat(artifactsToStrings(indexAction.getOutputs())) + .containsAtLeast( + "bin pkg/bin.lto/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o.imports", + "bin pkg/bin.lto/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o.thinlto.bc", + "bin pkg/bin.lto/" + rootExecPath + "/pkg/_objs/lib/libfile.pic.o.imports", + "bin pkg/bin.lto/" + rootExecPath + "/pkg/_objs/lib/libfile.pic.o.thinlto.bc", + "bin pkg/bin-lto-final.params"); + + assertThat(indexAction.getMnemonic()).isEqualTo("CppLTOIndexing"); + + assertThat(artifactsToStrings(indexAction.getInputs())) + .containsAtLeast( + "bin pkg/_objs/bin/binfile.pic.indexing.o", "bin pkg/_objs/lib/libfile.pic.indexing.o"); + + CppCompileAction bitcodeAction = + (CppCompileAction) + getPredecessorByInputName(indexAction, "pkg/_objs/bin/binfile.pic.indexing.o"); + assertThat(bitcodeAction.getMnemonic()).isEqualTo("CppCompile"); + assertThat(bitcodeAction.getArguments()) + .contains("lto_indexing_bitcode=" + prefix + "/bin/pkg/_objs/bin/binfile.pic.indexing.o"); + } + + @Test + public void testLinkshared() throws Exception { + targetName = "bin.so"; + createBuildFiles("linkshared = 1,"); + setupThinLTOCrosstool(CppRuleClasses.SUPPORTS_PIC); + useConfiguration(); + + CppLinkAction linkAction = getLinkAction(); + String rootExecPath = getRootExecPath(); + + Action backendAction = + getPredecessorByInputName( + linkAction, "pkg/bin.so.lto-obj/" + rootExecPath + "/pkg/_objs/bin.so/binfile.pic.o"); + assertThat(backendAction.getMnemonic()).isEqualTo("CcLtoBackendCompile"); + } + + @Test + public void testNoLinkstatic() throws Exception { + createBuildFiles("linkstatic = 0,"); + setupThinLTOCrosstool( + CppRuleClasses.SUPPORTS_DYNAMIC_LINKER, + CppRuleClasses.SUPPORTS_PIC, + CppRuleClasses.SUPPORTS_INTERFACE_SHARED_LIBRARIES); + useConfiguration("--noincompatible_make_thinlto_command_lines_standalone"); + + /* + We follow the chain from the final product backwards to verify intermediate actions. + + binary <=[Link]= + .ifso <=[SolibSymlink]= + _S...ifso <=[SolibSymlink]= + .ifso <=[Link]= + .lto-obj/...o <=[LTOBackend]= + {.o.thinlto.bc,.o.imports} <=[LTOIndexing]= + .o <= [CppCompile] .cc + */ + CppLinkAction linkAction = getLinkAction(); + String rootExecPath = getRootExecPath(); + + List commandLine = linkAction.getLinkCommandLineForTesting().getRawLinkArgv(); + String prefix = getTargetConfiguration().getOutputDirectory(RepositoryName.MAIN) + .getExecPathString(); + + assertThat(commandLine).contains("-Wl,@" + prefix + "/bin/pkg/bin-lto-final.params"); + + // We have no bitcode files: all files have pkg/bin.lto/ + for (String arg : commandLine) { + if (arg.contains("_objs") && !arg.contains("linkstamp.o")) { + assertThat(arg).contains("pkg/bin.lto"); + } + } + + assertThat(artifactsToStrings(linkAction.getInputs())) + .containsAtLeast( + "bin pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o", + "bin _solib_k8/libpkg_Sliblib.ifso", + "bin pkg/bin-2.params", + "bin pkg/bin-lto-final.params"); + + SolibSymlinkAction solibSymlinkAction = + (SolibSymlinkAction) getPredecessorByInputName(linkAction, "_solib_k8/libpkg_Sliblib.ifso"); + assertThat(solibSymlinkAction.getMnemonic()).isEqualTo("SolibSymlink"); + + CppLinkAction libLinkAction = + (CppLinkAction) getPredecessorByInputName(solibSymlinkAction, "bin/pkg/liblib.ifso"); + assertThat(libLinkAction.getMnemonic()).isEqualTo("CppLink"); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + libLinkAction, + "pkg/liblib.so.lto-obj/" + rootExecPath + "/pkg/_objs/lib/libfile.pic.o"); + assertThat(backendAction.getMnemonic()).isEqualTo("CcLtoBackendCompile"); + + assertThat(artifactsToStrings(backendAction.getInputs())) + .contains( + "bin pkg/liblib.so.lto/" + rootExecPath + "/pkg/_objs/lib/libfile.pic.o.thinlto.bc"); + + assertThat(backendAction.getArguments()) + .containsAtLeast( + "thinlto_index=" + + prefix + + "/bin/pkg/liblib.so.lto/" + + rootExecPath + + "/pkg/_objs/lib/libfile.pic.o.thinlto.bc", + "thinlto_output_object_file=" + + prefix + + "/bin/pkg/liblib.so.lto-obj/" + + rootExecPath + + "/pkg/_objs/lib/libfile.pic.o", + "thinlto_input_bitcode_file=" + prefix + "/bin/pkg/_objs/lib/libfile.pic.o"); + + CppLinkAction indexAction = + (CppLinkAction) + getPredecessorByInputName( + backendAction, + "pkg/liblib.so.lto/" + rootExecPath + "/pkg/_objs/lib/libfile.pic.o.thinlto.bc"); + + assertThat(indexAction.getArguments()) + .containsAtLeast( + "param_file=" + prefix + "/bin/pkg/liblib.so-lto-final.params", + "prefix_replace=" + + ";" + + prefix + + "/bin/pkg/liblib.so.lto/" + + ";" + + prefix + + "/bin/pkg/liblib.so.lto-obj/", + "object_suffix_replace=.indexing.o;.o"); + + assertThat(artifactsToStrings(indexAction.getOutputs())) + .containsAtLeast( + "bin pkg/liblib.so.lto/" + rootExecPath + "/pkg/_objs/lib/libfile.pic.o.imports", + "bin pkg/liblib.so.lto/" + rootExecPath + "/pkg/_objs/lib/libfile.pic.o.thinlto.bc", + "bin pkg/liblib.so-lto-final.params"); + + assertThat(indexAction.getMnemonic()).isEqualTo("CppLTOIndexing"); + + assertThat(artifactsToStrings(indexAction.getInputs())) + .contains("bin pkg/_objs/lib/libfile.pic.indexing.o"); + + CppCompileAction bitcodeAction = + (CppCompileAction) + getPredecessorByInputName(indexAction, "pkg/_objs/lib/libfile.pic.indexing.o"); + assertThat(bitcodeAction.getMnemonic()).isEqualTo("CppCompile"); + assertThat(bitcodeAction.getArguments()) + .contains("lto_indexing_bitcode=" + prefix + "/bin/pkg/_objs/lib/libfile.pic.indexing.o"); + } + + /** Helper method to get the root prefix from the given dwpFile. */ + private static PathFragment dwpRootPrefix(Artifact dwpFile) throws Exception { + return dwpFile + .getExecPath() + .subFragment( + 0, dwpFile.getExecPath().segmentCount() - dwpFile.getRootRelativePath().segmentCount()); + } + + /** Helper method that checks that a .dwp has the expected generating action structure. */ + private void validateDwp( + RuleContext ruleContext, + Artifact dwpFile, + CcToolchainProvider toolchain, + List expectedInputs) + throws Exception { + SpawnAction dwpAction = (SpawnAction) getGeneratingAction(dwpFile); + String dwpToolPath = toolchain.getToolPathFragment(Tool.DWP, ruleContext).getPathString(); + assertThat(dwpAction.getMnemonic()).isEqualTo("CcGenerateDwp"); + assertThat(dwpToolPath).isEqualTo(dwpAction.getCommandFilename()); + List commandArgs = dwpAction.getArguments(); + // The first argument should be the command being executed. + assertThat(dwpToolPath).isEqualTo(commandArgs.get(0)); + // The final two arguments should be "-o dwpOutputFile". + assertThat(commandArgs.subList(commandArgs.size() - 2, commandArgs.size())) + .containsExactly("-o", dwpFile.getExecPathString()) + .inOrder(); + // The remaining arguments should be the set of .dwo inputs (in any order). + assertThat(commandArgs.subList(1, commandArgs.size() - 2)) + .containsExactlyElementsIn(expectedInputs); + } + + @Test + public void testFission() throws Exception { + createBuildFiles(); + setupThinLTOCrosstool(CppRuleClasses.SUPPORTS_PIC, CppRuleClasses.PER_OBJECT_DEBUG_INFO); + useConfiguration("--fission=yes", "--copt=-g0"); + + String rootExecPath = getRootExecPath(); + LtoBackendAction backendAction = + getBackendAction("pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o"); + assertThat(backendAction.getMnemonic()).isEqualTo("CcLtoBackendCompile"); + assertThat(artifactsToStrings(backendAction.getOutputs())) + .containsExactly( + "bin pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o", + "bin pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.dwo"); + + assertThat(backendAction.getArguments()).containsAtLeast("-g0", "per_object_debug_info_option"); + + backendAction = + getBackendAction("pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/lib/libfile.pic.o"); + assertThat(backendAction.getMnemonic()).isEqualTo("CcLtoBackendCompile"); + assertThat(artifactsToStrings(backendAction.getOutputs())) + .containsExactly( + "bin pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/lib/libfile.pic.o", + "bin pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/lib/libfile.pic.dwo"); + + assertThat(backendAction.getArguments()).contains("per_object_debug_info_option"); + + // Now check the dwp action. + ConfiguredTarget pkg = getCurrentTarget(); + Artifact dwpFile = getFileConfiguredTarget(pkg.getLabel() + ".dwp").getArtifact(); + PathFragment rootPrefix = dwpRootPrefix(dwpFile); + RuleContext ruleContext = getRuleContext(pkg); + CcToolchainProvider toolchain = + CppHelper.getToolchainUsingDefaultCcToolchainAttribute(ruleContext); + validateDwp( + ruleContext, + dwpFile, + toolchain, + ImmutableList.of( + rootPrefix + "/pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/lib/libfile.pic.dwo", + rootPrefix + "/pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.dwo")); + } + + @Test + public void testNoLinkstaticFission() throws Exception { + createBuildFiles("linkstatic = 0,"); + setupThinLTOCrosstool( + CppRuleClasses.SUPPORTS_PIC, + CppRuleClasses.SUPPORTS_INTERFACE_SHARED_LIBRARIES, + CppRuleClasses.SUPPORTS_DYNAMIC_LINKER, + CppRuleClasses.PER_OBJECT_DEBUG_INFO); + useConfiguration("--fission=yes"); + + ConfiguredTarget pkg = getConfiguredTarget("//pkg:bin"); + Artifact pkgArtifact = getFilesToBuild(pkg).getSingleton(); + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(pkgArtifact); + String rootExecPath = pkgArtifact.getRoot().getExecPathString(); + + SolibSymlinkAction solibSymlinkAction = + (SolibSymlinkAction) getPredecessorByInputName(linkAction, "_solib_k8/libpkg_Sliblib.ifso"); + assertThat(solibSymlinkAction.getMnemonic()).isEqualTo("SolibSymlink"); + + CppLinkAction libLinkAction = + (CppLinkAction) getPredecessorByInputName(solibSymlinkAction, "bin/pkg/liblib.ifso"); + assertThat(libLinkAction.getMnemonic()).isEqualTo("CppLink"); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + libLinkAction, + "pkg/liblib.so.lto-obj/" + rootExecPath + "/pkg/_objs/lib/libfile.pic.o"); + assertThat(backendAction.getMnemonic()).isEqualTo("CcLtoBackendCompile"); + assertThat(artifactsToStrings(backendAction.getOutputs())) + .containsExactly( + "bin pkg/liblib.so.lto-obj/" + rootExecPath + "/pkg/_objs/lib/libfile.pic.o", + "bin pkg/liblib.so.lto-obj/" + rootExecPath + "/pkg/_objs/lib/libfile.pic.dwo"); + + assertThat(backendAction.getArguments()).contains("per_object_debug_info_option"); + + // Check the dwp action. + Artifact dwpFile = getFileConfiguredTarget(pkg.getLabel() + ".dwp").getArtifact(); + PathFragment rootPrefix = dwpRootPrefix(dwpFile); + RuleContext ruleContext = getRuleContext(pkg); + CcToolchainProvider toolchain = + CppHelper.getToolchainUsingDefaultCcToolchainAttribute(ruleContext); + validateDwp( + ruleContext, + dwpFile, + toolchain, + ImmutableList.of( + rootPrefix + "/pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.dwo")); + } + + @Test + public void testLinkstaticCcTestFission() throws Exception { + createTestFiles("linkstatic = 1,", ""); + + setupThinLTOCrosstool( + CppRuleClasses.SUPPORTS_PIC, + CppRuleClasses.THIN_LTO_LINKSTATIC_TESTS_USE_SHARED_NONLTO_BACKENDS, + CppRuleClasses.PER_OBJECT_DEBUG_INFO); + useConfiguration( + "--fission=yes", "--features=thin_lto_linkstatic_tests_use_shared_nonlto_backends"); + + ConfiguredTarget pkg = getConfiguredTarget("//pkg:bin_test"); + Artifact pkgArtifact = getFilesToBuild(pkg).getSingleton(); + String rootExecPath = pkgArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(pkgArtifact); + + // All backends should be shared non-LTO in this case + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, + "shared.nonlto-obj/" + rootExecPath + "/pkg/_objs/bin_test/bin_test.pic.o"); + assertThat(backendAction.getMnemonic()).isEqualTo("CcLtoBackendCompile"); + assertThat(artifactsToStrings(backendAction.getOutputs())) + .containsExactly( + "bin shared.nonlto-obj/" + rootExecPath + "/pkg/_objs/bin_test/bin_test.pic.o", + "bin shared.nonlto-obj/" + rootExecPath + "/pkg/_objs/bin_test/bin_test.pic.dwo"); + + assertThat(backendAction.getArguments()).contains("per_object_debug_info_option"); + + backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "shared.nonlto-obj/" + rootExecPath + "/pkg/_objs/lib/libfile.pic.o"); + assertThat(backendAction.getMnemonic()).isEqualTo("CcLtoBackendCompile"); + assertThat(backendAction.getArguments()).contains("-fPIC"); + assertThat(artifactsToStrings(backendAction.getOutputs())) + .containsExactly( + "bin shared.nonlto-obj/" + rootExecPath + "/pkg/_objs/lib/libfile.pic.o", + "bin shared.nonlto-obj/" + rootExecPath + "/pkg/_objs/lib/libfile.pic.dwo"); + + assertThat(backendAction.getArguments()).contains("per_object_debug_info_option"); + + // Now check the dwp action. + Artifact dwpFile = getFileConfiguredTarget(pkg.getLabel() + ".dwp").getArtifact(); + PathFragment rootPrefix = dwpRootPrefix(dwpFile); + RuleContext ruleContext = getRuleContext(pkg); + CcToolchainProvider toolchain = + CppHelper.getToolchainUsingDefaultCcToolchainAttribute(ruleContext); + validateDwp( + ruleContext, + dwpFile, + toolchain, + ImmutableList.of( + rootPrefix + "/shared.nonlto-obj/" + rootExecPath + "/pkg/_objs/lib/libfile.pic.dwo", + rootPrefix + + "/shared.nonlto-obj/" + + rootExecPath + + "/pkg/_objs/bin_test/bin_test.pic.dwo")); + } + + @Test + public void testLinkstaticCcTest() throws Exception { + createTestFiles("linkstatic = 1,", ""); + + setupThinLTOCrosstool( + CppRuleClasses.SUPPORTS_PIC, + CppRuleClasses.THIN_LTO_LINKSTATIC_TESTS_USE_SHARED_NONLTO_BACKENDS, + CppRuleClasses.PER_OBJECT_DEBUG_INFO); + useConfiguration("--features=thin_lto_linkstatic_tests_use_shared_nonlto_backends"); + + ConfiguredTarget pkg = getConfiguredTarget("//pkg:bin_test"); + Artifact pkgArtifact = getFilesToBuild(pkg).getSingleton(); + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(pkgArtifact); + + ConfiguredTarget pkg2 = getConfiguredTarget("//pkg:bin_test2"); + Artifact pkgArtifact2 = getFilesToBuild(pkg2).getSingleton(); + CppLinkAction linkAction2 = (CppLinkAction) getGeneratingAction(pkgArtifact2); + + // All backends should be shared non-LTO in this case + String rootExecPath1 = pkgArtifact.getRoot().getExecPathString(); + String rootExecPath2 = pkgArtifact.getRoot().getExecPathString(); + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, + "shared.nonlto-obj/" + rootExecPath1 + "/pkg/_objs/bin_test/bin_test.pic.o"); + assertThat(backendAction.getMnemonic()).isEqualTo("CcLtoBackendCompile"); + + backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "shared.nonlto-obj/" + rootExecPath1 + "/pkg/_objs/lib/libfile.pic.o"); + assertThat(backendAction.getMnemonic()).isEqualTo("CcLtoBackendCompile"); + assertThat(backendAction.getArguments()).contains("-fPIC"); + + LtoBackendAction backendAction2 = + (LtoBackendAction) + getPredecessorByInputName( + linkAction2, "shared.nonlto-obj/" + rootExecPath2 + "/pkg/_objs/lib/libfile.pic.o"); + assertThat(backendAction2.getMnemonic()).isEqualTo("CcLtoBackendCompile"); + + assertThat(backendAction).isEqualTo(backendAction2); + } + + @Test + public void testTestOnlyTarget() throws Exception { + createBuildFiles("testonly = 1,"); + + setupThinLTOCrosstool( + CppRuleClasses.SUPPORTS_PIC, + CppRuleClasses.THIN_LTO_LINKSTATIC_TESTS_USE_SHARED_NONLTO_BACKENDS); + useConfiguration("--features=thin_lto_linkstatic_tests_use_shared_nonlto_backends"); + + ConfiguredTarget pkg = getConfiguredTarget("//pkg:bin"); + Artifact pkgArtifact = getFilesToBuild(pkg).getSingleton(); + String rootExecPath = pkgArtifact.getRoot().getExecPathString(); + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(pkgArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "shared.nonlto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o"); + assertThat(backendAction.getMnemonic()).isEqualTo("CcLtoBackendCompile"); + } + + @Test + public void testUseSharedAllLinkstatic() throws Exception { + createBuildFiles(); + + setupThinLTOCrosstool( + CppRuleClasses.THIN_LTO_ALL_LINKSTATIC_USE_SHARED_NONLTO_BACKENDS, + CppRuleClasses.SUPPORTS_PIC); + useConfiguration("--features=thin_lto_all_linkstatic_use_shared_nonlto_backends"); + + ConfiguredTarget pkg = getConfiguredTarget("//pkg:bin"); + Artifact pkgArtifact = getFilesToBuild(pkg).getSingleton(); + String rootExecPath = pkgArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(pkgArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "shared.nonlto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o"); + assertThat(backendAction.getMnemonic()).isEqualTo("CcLtoBackendCompile"); + } + + private Action getPredecessorByInputName(Action action, String str) { + for (Artifact a : action.getInputs().toList()) { + if (a.getExecPathString().contains(str)) { + return getGeneratingAction(a); + } + } + return null; + } + + @Test + public void testFdoInstrument() throws Exception { + scratch.file( + "pkg/BUILD", + "package(features = ['thin_lto', 'use_lto_native_object_directory'])", + "", + "cc_binary(name = 'bin',", + " srcs = ['binfile.cc', ],", + " malloc = '//base:system_malloc')"); + + scratch.file("pkg/binfile.cc", "int main() {}"); + + setupThinLTOCrosstool(CppRuleClasses.SUPPORTS_PIC, CppRuleClasses.FDO_INSTRUMENT); + useConfiguration("--fdo_instrument=profiles"); + + ConfiguredTarget pkg = getConfiguredTarget("//pkg:bin"); + + Artifact pkgArtifact = getFilesToBuild(pkg).getSingleton(); + String rootExecPath = pkgArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(pkgArtifact); + assertThat(linkAction.getOutputs()).containsExactly(pkgArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o"); + // If the LtoBackendAction incorrectly tries to add the fdo_instrument + // feature, we will fail with an "unknown variable 'fdo_instrument_path'" + // error. But let's also explicitly confirm that the fdo_instrument + // option didn't end up here. + assertThat(backendAction.getArguments()).doesNotContain("fdo_instrument_option"); + } + + @Test + public void testLtoIndexOpt() throws Exception { + createBuildFiles(); + + setupThinLTOCrosstool(CppRuleClasses.SUPPORTS_PIC); + useConfiguration( + "--ltoindexopt=anltoindexopt", "--noincompatible_make_thinlto_command_lines_standalone"); + + /* + We follow the chain from the final product backwards. + + binary <=[Link]= + .lto-obj/...o <=[LTOBackend]= + {.o.thinlto.bc,.o.imports} <=[LTOIndexing]= + .o <= [CppCompile] .cc + */ + ConfiguredTarget pkg = getConfiguredTarget("//pkg:bin"); + + Artifact pkgArtifact = getFilesToBuild(pkg).getSingleton(); + String rootExecPath = pkgArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(pkgArtifact); + assertThat(linkAction.getOutputs()).containsExactly(pkgArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o"); + assertThat(backendAction.getMnemonic()).isEqualTo("CcLtoBackendCompile"); + + CppLinkAction indexAction = + (CppLinkAction) + getPredecessorByInputName( + backendAction, + "pkg/bin.lto/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o.thinlto.bc"); + + assertThat(indexAction.getArguments()).contains("anltoindexopt"); + } + + @Test + public void testLtoStandaloneCommandLines() throws Exception { + createBuildFiles(); + + setupThinLTOCrosstool(CppRuleClasses.SUPPORTS_PIC); + useConfiguration( + "--ltoindexopt=anltoindexopt", + "--incompatible_make_thinlto_command_lines_standalone", + "--features=thin_lto", + "--features=use_lto_native_object_directory"); + + /* + We follow the chain from the final product backwards. + + binary <=[Link]= + .lto-obj/...o <=[LTOBackend]= + {.o.thinlto.bc,.o.imports} <=[LTOIndexing]= + .o <= [CppCompile] .cc + */ + ConfiguredTarget pkg = getConfiguredTarget("//pkg:bin"); + + Artifact pkgArtifact = getFilesToBuild(pkg).getSingleton(); + String rootExecPath = pkgArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(pkgArtifact); + assertThat(linkAction.getOutputs()).containsExactly(pkgArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o"); + assertThat(backendAction.getMnemonic()).isEqualTo("CcLtoBackendCompile"); + + CppLinkAction indexAction = + (CppLinkAction) + getPredecessorByInputName( + backendAction, + "pkg/bin.lto/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o.thinlto.bc"); + + assertThat(indexAction.getArguments()) + .contains("--i_come_from_standalone_lto_index=anltoindexopt"); + } + + @Test + public void testCopt() throws Exception { + createBuildFiles(); + + setupThinLTOCrosstool(CppRuleClasses.SUPPORTS_PIC); + useConfiguration("--copt=acopt"); + + /* + We follow the chain from the final product backwards. + + binary <=[Link]= + .lto-obj/...o <=[LTOBackend]= + */ + ConfiguredTarget pkg = getConfiguredTarget("//pkg:bin"); + + Artifact pkgArtifact = getFilesToBuild(pkg).getSingleton(); + String rootExecPath = pkgArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(pkgArtifact); + assertThat(linkAction.getOutputs()).containsExactly(pkgArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o"); + assertThat(backendAction.getMnemonic()).isEqualTo("CcLtoBackendCompile"); + assertThat(backendAction.getArguments()).contains("acopt"); + } + + @Test + public void testPerFileCopt() throws Exception { + createBuildFiles(); + setupThinLTOCrosstool(CppRuleClasses.SUPPORTS_PIC); + useConfiguration( + "--per_file_copt=binfile\\.cc@copt1", + "--per_file_copt=libfile\\.cc@copt2", + "--per_file_copt=.*\\.cc,-binfile\\.cc@copt2"); + + /* + We follow the chain from the final product backwards. + + binary <=[Link]= + .lto-obj/...o <=[LTOBackend]= + */ + ConfiguredTarget pkg = getConfiguredTarget("//pkg:bin"); + Artifact pkgArtifact = getFilesToBuild(pkg).getSingleton(); + String rootExecPath = pkgArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(pkgArtifact); + assertThat(linkAction.getOutputs()).containsExactly(pkgArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o"); + assertThat(backendAction.getArguments()).contains("copt1"); + assertThat(backendAction.getArguments()).doesNotContain("copt2"); + + backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/lib/libfile.pic.o"); + assertThat(backendAction.getArguments()).doesNotContain("copt1"); + assertThat(backendAction.getArguments()).contains("copt2"); + } + + @Test + public void testCoptNoCoptAttributes() throws Exception { + createBuildFiles("copts = ['acopt', 'nocopt1'], nocopts = 'nocopt1|nocopt2',"); + + setupThinLTOCrosstool(CppRuleClasses.SUPPORTS_PIC); + useConfiguration("--copt=nocopt2", "--noincompatible_disable_nocopts"); + + /* + We follow the chain from the final product backwards. + + binary <=[Link]= + .lto-obj/...o <=[LTOBackend]= + */ + ConfiguredTarget pkg = getConfiguredTarget("//pkg:bin"); + + Artifact pkgArtifact = getFilesToBuild(pkg).getSingleton(); + String rootExecPath = pkgArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(pkgArtifact); + assertThat(linkAction.getOutputs()).containsExactly(pkgArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o"); + assertThat(backendAction.getMnemonic()).isEqualTo("CcLtoBackendCompile"); + assertThat(backendAction.getArguments()).contains("acopt"); + // TODO(b/122303926): Remove when nocopts are removed, or uncomment and fix if not removing. + // assertThat(backendAction.getArguments()).doesNotContain("nocopt1"); + // assertThat(backendAction.getArguments()).doesNotContain("nocopt2"); + } + + @Test + public void testLtoBackendOpt() throws Exception { + createBuildFiles(); + + setupThinLTOCrosstool(CppRuleClasses.SUPPORTS_PIC, MockCcSupport.USER_COMPILE_FLAGS); + useConfiguration("--ltobackendopt=anltobackendopt"); + + /* + We follow the chain from the final product backwards. + + binary <=[Link]= + .lto-obj/...o <=[LTOBackend]= + */ + ConfiguredTarget pkg = getConfiguredTarget("//pkg:bin"); + + Artifact pkgArtifact = getFilesToBuild(pkg).getSingleton(); + String rootExecPath = pkgArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(pkgArtifact); + assertThat(linkAction.getOutputs()).containsExactly(pkgArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o"); + assertThat(backendAction.getMnemonic()).isEqualTo("CcLtoBackendCompile"); + assertThat(backendAction.getArguments()) + .containsAtLeast("--default-compile-flag", "anltobackendopt"); + } + + @Test + public void testPerFileLtoBackendOpt() throws Exception { + createBuildFiles(); + + setupThinLTOCrosstool(CppRuleClasses.SUPPORTS_PIC); + useConfiguration( + "--per_file_ltobackendopt=binfile\\.pic\\.o@ltobackendopt1", + "--per_file_ltobackendopt=.*\\.o,-binfile\\.pic\\.o@ltobackendopt2"); + + /* + We follow the chain from the final product backwards. + + binary <=[Link]= + .lto-obj/...o <=[LTOBackend]= + */ + ConfiguredTarget pkg = getConfiguredTarget("//pkg:bin"); + Artifact pkgArtifact = getFilesToBuild(pkg).getSingleton(); + String rootExecPath = pkgArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(pkgArtifact); + assertThat(linkAction.getOutputs()).containsExactly(pkgArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o"); + assertThat(backendAction.getArguments()).contains("ltobackendopt1"); + assertThat(backendAction.getArguments()).doesNotContain("ltobackendopt2"); + + backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/lib/libfile.pic.o"); + assertThat(backendAction.getArguments()).doesNotContain("ltobackendopt1"); + assertThat(backendAction.getArguments()).contains("ltobackendopt2"); + } + + @Test + public void testNoUseLtoIndexingBitcodeFile() throws Exception { + createBuildFiles(); + + setupThinLTOCrosstool( + CppRuleClasses.NO_USE_LTO_INDEXING_BITCODE_FILE, CppRuleClasses.SUPPORTS_PIC); + useConfiguration( + "--features=no_use_lto_indexing_bitcode_file", + "--features=use_lto_native_object_directory"); + String rootExecPath = getRootExecPath(); + + /* + We follow the chain from the final product backwards. + + binary <=[Link]= + .lto-obj/...o <=[LTOBackend]= + {.o.thinlto.bc,.o.imports} <=[LTOIndexing]= + .o <= [CppCompile] .cc + */ + CppLinkAction indexAction = + getIndexAction( + getBackendAction("pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o")); + + assertThat(indexAction.getArguments()).doesNotContain("object_suffix_replace"); + + assertThat(artifactsToStrings(indexAction.getInputs())) + .containsAtLeast("bin pkg/_objs/bin/binfile.pic.o", "bin pkg/_objs/lib/libfile.pic.o"); + + CppCompileAction bitcodeAction = + (CppCompileAction) getPredecessorByInputName(indexAction, "pkg/_objs/bin/binfile.pic.o"); + assertThat(bitcodeAction.getArguments()).doesNotContain("lto_indexing_bitcode="); + } + + @Test + public void testAutoFdo() throws Exception { + scratch.file( + "pkg/BUILD", + "package(features = ['thin_lto', 'use_lto_native_object_directory'])", + "", + "cc_binary(name = 'bin',", + " srcs = ['binfile.cc', ],", + " malloc = '//base:system_malloc')"); + + scratch.file("pkg/binfile.cc", "int main() {}"); + scratch.file("pkg/profile.afdo", ""); + + setupThinLTOCrosstool(CppRuleClasses.AUTOFDO); + useConfiguration("--fdo_optimize=/pkg/profile.afdo", "--compilation_mode=opt"); + + Artifact binArtifact = getFilesToBuild(getConfiguredTarget("//pkg:bin")).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + assertThat(linkAction.getOutputs()).containsExactly(binArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + + // Checks that -fauto-profile is added to the LtoBackendAction. + assertThat(Joiner.on(" ").join(backendAction.getArguments())).containsMatch( + "-fauto-profile=[^ ]*/profile.afdo"); + assertThat(ActionsTestUtil.baseArtifactNames(backendAction.getInputs())).contains( + "profile.afdo"); + } + + private void setupThinLTOCrosstool(String... extraFeatures) throws Exception { + String[] allFeatures = + Stream.concat( + Stream.of( + CppRuleClasses.THIN_LTO, + CppRuleClasses.USE_LTO_NATIVE_OBJECT_DIRECTORY, + CppRuleClasses.SUPPORTS_START_END_LIB, + MockCcSupport.HOST_AND_NONHOST_CONFIGURATION_FEATURES), + stream(extraFeatures)) + .toArray(String[]::new); + AnalysisMock.get() + .ccSupport() + .setupCcToolchainConfig( + mockToolsConfig, CcToolchainConfig.builder().withFeatures(allFeatures)); + } + + private void setupAutoFdoThinLtoCrosstool() throws Exception { + setupThinLTOCrosstool( + CppRuleClasses.AUTOFDO, + CppRuleClasses.ENABLE_AFDO_THINLTO, + CppRuleClasses.AUTOFDO_IMPLICIT_THINLTO); + } + + /** + * Tests that ThinLTO is not enabled for AFDO with LLVM without + * --features=autofdo_implicit_thinlto. + */ + @Test + public void testAutoFdoNoImplicitThinLto() throws Exception { + scratch.file( + "pkg/BUILD", + "", + "cc_binary(name = 'bin',", + " srcs = ['binfile.cc', ],", + " malloc = '//base:system_malloc')"); + + scratch.file("pkg/binfile.cc", "int main() {}"); + scratch.file("pkg/profile.afdo", ""); + + setupAutoFdoThinLtoCrosstool(); + useConfiguration("--fdo_optimize=/pkg/profile.afdo", "--compilation_mode=opt"); + + Artifact binArtifact = getFilesToBuild(getConfiguredTarget("//pkg:bin")).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + assertThat(linkAction.getOutputs()).containsExactly(binArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + // We should not have a ThinLTO backend action + assertThat(backendAction).isNull(); + } + + /** Tests that --features=autofdo_implicit_thinlto enables ThinLTO for AFDO with LLVM. */ + @Test + public void testAutoFdoImplicitThinLto() throws Exception { + scratch.file( + "pkg/BUILD", + "", + "cc_binary(name = 'bin',", + " srcs = ['binfile.cc', ],", + " malloc = '//base:system_malloc')"); + + scratch.file("pkg/binfile.cc", "int main() {}"); + scratch.file("pkg/profile.afdo", ""); + + setupAutoFdoThinLtoCrosstool(); + useConfiguration( + "--fdo_optimize=/pkg/profile.afdo", + "--compilation_mode=opt", + "--features=autofdo_implicit_thinlto", + "--features=use_lto_native_object_directory"); + + Artifact binArtifact = getFilesToBuild(getConfiguredTarget("//pkg:bin")).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + assertThat(linkAction.getOutputs()).containsExactly(binArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + // For ThinLTO compilation we should have a non-null backend action + assertThat(backendAction).isNotNull(); + } + + /** + * Tests that --features=-thin_lto overrides --features=autofdo_implicit_thinlto and prevents + * enabling ThinLTO for AFDO with LLVM. + */ + @Test + public void testAutoFdoImplicitThinLtoDisabledOption() throws Exception { + scratch.file( + "pkg/BUILD", + "", + "cc_binary(name = 'bin',", + " srcs = ['binfile.cc', ],", + " malloc = '//base:system_malloc')"); + + scratch.file("pkg/binfile.cc", "int main() {}"); + scratch.file("pkg/profile.afdo", ""); + + setupAutoFdoThinLtoCrosstool(); + useConfiguration( + "--fdo_optimize=/pkg/profile.afdo", + "--compilation_mode=opt", + "--features=autofdo_implicit_thinlto", + "--features=-thin_lto", + "--features=use_lto_native_object_directory"); + + Artifact binArtifact = getFilesToBuild(getConfiguredTarget("//pkg:bin")).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + assertThat(linkAction.getOutputs()).containsExactly(binArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + // We should not have a ThinLTO backend action + assertThat(backendAction).isNull(); + } + + /** + * Tests that features=[-thin_lto] in the build rule overrides --features=autofdo_implicit_thinlto + * and prevents enabling ThinLTO for AFDO with LLVM. + */ + @Test + public void testAutoFdoImplicitThinLtoDisabledRule() throws Exception { + scratch.file( + "pkg/BUILD", + "", + "cc_binary(name = 'bin',", + " srcs = ['binfile.cc', ],", + " features = ['-thin_lto', 'use_lto_native_object_directory'],", + " malloc = '//base:system_malloc')"); + + scratch.file("pkg/binfile.cc", "int main() {}"); + scratch.file("pkg/profile.afdo", ""); + + setupAutoFdoThinLtoCrosstool(); + useConfiguration( + "--fdo_optimize=/pkg/profile.afdo", + "--compilation_mode=opt", + "--features=autofdo_implicit_thinlto"); + + Artifact binArtifact = getFilesToBuild(getConfiguredTarget("//pkg:bin")).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + assertThat(linkAction.getOutputs()).containsExactly(binArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + // We should not have a ThinLTO backend action + assertThat(backendAction).isNull(); + } + + /** + * Tests that features=[-thin_lto] in the package overrides --features=autofdo_implicit_thinlto + * and prevents enabling ThinLTO for AFDO with LLVM. + */ + @Test + public void testAutoFdoImplicitThinLtoDisabledPackage() throws Exception { + scratch.file( + "pkg/BUILD", + "package(features = ['-thin_lto', 'use_lto_native_object_directory'])", + "", + "cc_binary(name = 'bin',", + " srcs = ['binfile.cc', ],", + " malloc = '//base:system_malloc')"); + + scratch.file("pkg/binfile.cc", "int main() {}"); + scratch.file("pkg/profile.afdo", ""); + + setupAutoFdoThinLtoCrosstool(); + useConfiguration( + "--fdo_optimize=/pkg/profile.afdo", + "--compilation_mode=opt", + "--features=autofdo_implicit_thinlto"); + + Artifact binArtifact = getFilesToBuild(getConfiguredTarget("//pkg:bin")).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + assertThat(linkAction.getOutputs()).containsExactly(binArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + // We should not have a ThinLTO backend action + assertThat(backendAction).isNull(); + } + + private void setupFdoThinLtoCrosstool() throws Exception { + setupThinLTOCrosstool( + CppRuleClasses.FDO_OPTIMIZE, + CppRuleClasses.ENABLE_FDO_THINLTO, + MockCcSupport.FDO_IMPLICIT_THINLTO); + } + + /** + * Tests that ThinLTO is not enabled for FDO with LLVM without --features=fdo_implicit_thinlto. + */ + @Test + public void testFdoNoImplicitThinLto() throws Exception { + scratch.file( + "pkg/BUILD", + "", + "cc_binary(name = 'bin',", + " srcs = ['binfile.cc', ],", + " malloc = '//base:system_malloc')"); + + scratch.file("pkg/binfile.cc", "int main() {}"); + scratch.file("pkg/profile.zip", ""); + + setupFdoThinLtoCrosstool(); + useConfiguration("--fdo_optimize=/pkg/profile.zip", "--compilation_mode=opt"); + + Artifact binArtifact = getFilesToBuild(getConfiguredTarget("//pkg:bin")).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + assertThat(linkAction.getOutputs()).containsExactly(binArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + // We should not have a ThinLTO backend action + assertThat(backendAction).isNull(); + } + + /** Tests that --features=fdo_implicit_thinlto enables ThinLTO for FDO with LLVM. */ + @Test + public void testFdoImplicitThinLto() throws Exception { + scratch.file( + "pkg/BUILD", + "", + "cc_binary(name = 'bin',", + " srcs = ['binfile.cc', ],", + " malloc = '//base:system_malloc')"); + + scratch.file("pkg/binfile.cc", "int main() {}"); + scratch.file("pkg/profile.zip", ""); + + setupFdoThinLtoCrosstool(); + useConfiguration( + "--fdo_optimize=/pkg/profile.zip", + "--compilation_mode=opt", + "--features=fdo_implicit_thinlto", + "--features=use_lto_native_object_directory"); + + Artifact binArtifact = getFilesToBuild(getConfiguredTarget("//pkg:bin")).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + assertThat(linkAction.getOutputs()).containsExactly(binArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + // For ThinLTO compilation we should have a non-null backend action + assertThat(backendAction).isNotNull(); + } + + /** + * Tests that --features=-thin_lto overrides --features=fdo_implicit_thinlto and prevents enabling + * ThinLTO for FDO with LLVM. + */ + @Test + public void testFdoImplicitThinLtoDisabledOption() throws Exception { + scratch.file( + "pkg/BUILD", + "", + "cc_binary(name = 'bin',", + " srcs = ['binfile.cc', ],", + " malloc = '//base:system_malloc')"); + + scratch.file("pkg/binfile.cc", "int main() {}"); + scratch.file("pkg/profile.zip", ""); + + setupFdoThinLtoCrosstool(); + useConfiguration( + "--fdo_optimize=/pkg/profile.zip", + "--compilation_mode=opt", + "--features=fdo_implicit_thinlto", + "--features=-thin_lto", + "--features=use_lto_native_object_directory"); + + Artifact binArtifact = getFilesToBuild(getConfiguredTarget("//pkg:bin")).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + assertThat(linkAction.getOutputs()).containsExactly(binArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + // We should not have a ThinLTO backend action + assertThat(backendAction).isNull(); + } + + /** + * Tests that features=[-thin_lto] in the build rule overrides --features=fdo_implicit_thinlto and + * prevents enabling ThinLTO for FDO with LLVM. + */ + @Test + public void testFdoImplicitThinLtoDisabledRule() throws Exception { + scratch.file( + "pkg/BUILD", + "", + "cc_binary(name = 'bin',", + " srcs = ['binfile.cc', ],", + " features = ['-thin_lto', 'use_lto_native_object_directory'],", + " malloc = '//base:system_malloc')"); + + scratch.file("pkg/binfile.cc", "int main() {}"); + scratch.file("pkg/profile.zip", ""); + + setupFdoThinLtoCrosstool(); + useConfiguration( + "--fdo_optimize=/pkg/profile.zip", + "--compilation_mode=opt", + "--features=fdo_implicit_thinlto"); + + Artifact binArtifact = getFilesToBuild(getConfiguredTarget("//pkg:bin")).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + assertThat(linkAction.getOutputs()).containsExactly(binArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + // We should not have a ThinLTO backend action + assertThat(backendAction).isNull(); + } + + /** + * Tests that features=[-thin_lto] in the package overrides --features=fdo_implicit_thinlto and + * prevents enabling ThinLTO for FDO with LLVM. + */ + @Test + public void testFdoImplicitThinLtoDisabledPackage() throws Exception { + setupThinLTOCrosstool(); + scratch.file( + "pkg/BUILD", + "package(features = ['-thin_lto', 'use_lto_native_object_directory'])", + "", + "cc_binary(name = 'bin',", + " srcs = ['binfile.cc', ],", + " malloc = '//base:system_malloc')"); + + scratch.file("pkg/binfile.cc", "int main() {}"); + scratch.file("pkg/profile.zip", ""); + + setupFdoThinLtoCrosstool(); + useConfiguration( + "--fdo_optimize=/pkg/profile.zip", + "--compilation_mode=opt", + "--features=fdo_implicit_thinlto"); + + Artifact binArtifact = getFilesToBuild(getConfiguredTarget("//pkg:bin")).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + assertThat(linkAction.getOutputs()).containsExactly(binArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + // We should not have a ThinLTO backend action + assertThat(backendAction).isNull(); + } + + private void setupXBinaryFdoThinLtoCrosstool() throws Exception { + setupThinLTOCrosstool( + CppRuleClasses.XBINARYFDO, + CppRuleClasses.ENABLE_XFDO_THINLTO, + MockCcSupport.XFDO_IMPLICIT_THINLTO); + } + + /** + * Tests that ThinLTO is not enabled for XFDO with LLVM without + * --features=xbinaryfdo_implicit_thinlto. + */ + @Test + public void testXBinaryFdoNoImplicitThinLto() throws Exception { + scratch.file( + "pkg/BUILD", + "", + "cc_binary(name = 'bin',", + " srcs = ['binfile.cc', ])", + "fdo_profile(name='out.xfdo', profile='profiles.xfdo')"); + + scratch.file("pkg/binfile.cc", "int main() {}"); + + setupXBinaryFdoThinLtoCrosstool(); + useConfiguration("--xbinary_fdo=//pkg:out.xfdo", "--compilation_mode=opt"); + + Artifact binArtifact = getFilesToBuild(getConfiguredTarget("//pkg:bin")).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + assertThat(linkAction.getOutputs()).containsExactly(binArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + // We should not have a ThinLTO backend action + assertThat(backendAction).isNull(); + } + + /** Tests that --features=xbinaryfdo_implicit_thinlto enables ThinLTO for XFDO with LLVM. */ + @Test + public void testXBinaryFdoImplicitThinLto() throws Exception { + scratch.file( + "pkg/BUILD", + "", + "cc_binary(name = 'bin',", + " srcs = ['binfile.cc', ])", + "fdo_profile(name='out.xfdo', profile='profiles.xfdo')"); + + scratch.file("pkg/binfile.cc", "int main() {}"); + + setupXBinaryFdoThinLtoCrosstool(); + useConfiguration( + "--xbinary_fdo=//pkg:out.xfdo", + "--compilation_mode=opt", + "--features=xbinaryfdo_implicit_thinlto", + "--features=use_lto_native_object_directory"); + + Artifact binArtifact = getFilesToBuild(getConfiguredTarget("//pkg:bin")).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + assertThat(linkAction.getOutputs()).containsExactly(binArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + // For ThinLTO compilation we should have a non-null backend action + assertThat(backendAction).isNotNull(); + } + + /** + * Tests that --features=-thin_lto overrides --features=xbinaryfdo_implicit_thinlto and prevents + * enabling ThinLTO for XFDO with LLVM. + */ + @Test + public void testXBinaryFdoImplicitThinLtoDisabledOption() throws Exception { + scratch.file( + "pkg/BUILD", + "", + "cc_binary(name = 'bin',", + " srcs = ['binfile.cc', ])", + "fdo_profile(name='out.xfdo', profile='profiles.xfdo')"); + + scratch.file("pkg/binfile.cc", "int main() {}"); + + setupXBinaryFdoThinLtoCrosstool(); + useConfiguration( + "--xbinary_fdo=//pkg:out.xfdo", + "--compilation_mode=opt", + "--features=xbinaryfdo_implicit_thinlto", + "--features=-thin_lto", + "--features=use_lto_native_object_directory"); + + Artifact binArtifact = getFilesToBuild(getConfiguredTarget("//pkg:bin")).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + assertThat(linkAction.getOutputs()).containsExactly(binArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + // We should not have a ThinLTO backend action + assertThat(backendAction).isNull(); + } + + /** + * Tests that features=[-thin_lto] in the build rule overrides + * --features=xbinaryfdo_implicit_thinlto and prevents enabling ThinLTO for XFDO with LLVM. + */ + @Test + public void testXBinaryFdoImplicitThinLtoDisabledRule() throws Exception { + scratch.file( + "pkg/BUILD", + "", + "cc_binary(name = 'bin',", + " srcs = ['binfile.cc', ],", + " features = ['-thin_lto', 'use_lto_native_object_directory'])", + "fdo_profile(name='out.xfdo', profile='profiles.xfdo')"); + + scratch.file("pkg/binfile.cc", "int main() {}"); + + setupXBinaryFdoThinLtoCrosstool(); + useConfiguration( + "--xbinary_fdo=//pkg:out.xfdo", + "--compilation_mode=opt", + "--features=xbinaryfdo_implicit_thinlto"); + + Artifact binArtifact = getFilesToBuild(getConfiguredTarget("//pkg:bin")).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + assertThat(linkAction.getOutputs()).containsExactly(binArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + // We should not have a ThinLTO backend action + assertThat(backendAction).isNull(); + } + + /** + * Tests that features=[-thin_lto] in the package overrides --features=fdo_implicit_thinlto and + * prevents enabling ThinLTO for XFDO with LLVM. + */ + @Test + public void testXBinaryFdoImplicitThinLtoDisabledPackage() throws Exception { + scratch.file( + "pkg/BUILD", + "package(features = ['-thin_lto', 'use_lto_native_object_directory'])", + "", + "cc_binary(name = 'bin',", + " srcs = ['binfile.cc', ])", + "fdo_profile(name='out.xfdo', profile='profiles.xfdo')"); + + scratch.file("pkg/binfile.cc", "int main() {}"); + + setupXBinaryFdoThinLtoCrosstool(); + useConfiguration( + "--xbinary_fdo=//pkg:out.xfdo", + "--compilation_mode=opt", + "--features=xbinaryfdo_implicit_thinlto"); + + Artifact binArtifact = getFilesToBuild(getConfiguredTarget("//pkg:bin")).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + assertThat(linkAction.getOutputs()).containsExactly(binArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + // We should not have a ThinLTO backend action + assertThat(backendAction).isNull(); + } + + @Test + public void testXBinaryFdo() throws Exception { + scratch.file( + "pkg/BUILD", + "package(features = ['thin_lto', 'use_lto_native_object_directory'])", + "", + "cc_binary(name = 'bin',", + " srcs = ['binfile.cc', ],", + " malloc = '//base:system_malloc')", + "fdo_profile(name='out.xfdo', profile='profiles.xfdo')"); + + scratch.file("pkg/binfile.cc", "int main() {}"); + + setupThinLTOCrosstool(CppRuleClasses.XBINARYFDO); + useConfiguration("--xbinary_fdo=//pkg:out.xfdo", "--compilation_mode=opt"); + + Artifact binArtifact = getFilesToBuild(getConfiguredTarget("//pkg:bin")).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + assertThat(linkAction.getOutputs()).containsExactly(binArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + + // Checks that -fauto-profile is added to the LtoBackendAction. + assertThat(Joiner.on(" ").join(backendAction.getArguments())) + .containsMatch("-fauto-profile=[^ ]*/profiles.xfdo"); + assertThat(ActionsTestUtil.baseArtifactNames(backendAction.getInputs())) + .contains("profiles.xfdo"); + } + + /** + * Tests that ThinLTO is not enabled for XBINARYFDO with --features=autofdo_implicit_thinlto and + * --features=fdo_implicit_thinlto. + */ + @Test + public void testXBinaryFdoNoAutoFdoOrFdoImplicitThinLto() throws Exception { + scratch.file( + "pkg/BUILD", + "", + "cc_binary(name = 'bin',", + " srcs = ['binfile.cc', ],", + " malloc = '//base:system_malloc')", + "fdo_profile(name='out.xfdo', profile='profiles.xfdo')"); + + scratch.file("pkg/binfile.cc", "int main() {}"); + + setupThinLTOCrosstool( + CppRuleClasses.ENABLE_FDO_THINLTO, + MockCcSupport.FDO_IMPLICIT_THINLTO, + CppRuleClasses.ENABLE_AFDO_THINLTO, + MockCcSupport.AUTOFDO_IMPLICIT_THINLTO, + CppRuleClasses.XBINARYFDO); + useConfiguration( + "--xbinary_fdo=//pkg:out.xfdo", + "--compilation_mode=opt", + "--features=autofdo_implicit_thinlto", + "--features=fdo_implicit_thinlto", + "--features=use_lto_native_object_directory"); + + Artifact binArtifact = getFilesToBuild(getConfiguredTarget("//pkg:bin")).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + assertThat(linkAction.getOutputs()).containsExactly(binArtifact); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/pkg/binfile.o"); + // We should not have a ThinLTO backend action + assertThat(backendAction).isNull(); + } + + @Test + public void testPICBackendOrder() throws Exception { + createBuildFiles(); + + setupThinLTOCrosstool(CppRuleClasses.SUPPORTS_PIC); + useConfiguration("--copt=-fno-PIE"); + String rootExecPath = getRootExecPath(); + LtoBackendAction backendAction = + getBackendAction("pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.pic.o"); + assertThat(backendAction.getMnemonic()).isEqualTo("CcLtoBackendCompile"); + assertThat(backendAction.getArguments()).containsAtLeast("-fno-PIE", "-fPIC").inOrder(); + } + + @Test + public void testPropellerOptimizeAbsoluteOptions() throws Exception { + createBuildFiles(); + + setupThinLTOCrosstool(CppRuleClasses.SUPPORTS_PIC, CppRuleClasses.AUTOFDO); + + useConfiguration( + "--propeller_optimize_absolute_cc_profile=/tmp/cc_profile.txt", + "--propeller_optimize_absolute_ld_profile=/tmp/ld_profile.txt", + "--compilation_mode=opt"); + Artifact binArtifact = getFilesToBuild(getCurrentTarget()).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + assertThat(linkAction.getOutputs()).containsExactly(binArtifact); + assertThat(ActionsTestUtil.baseArtifactNames(linkAction.getInputs())) + .contains("ld_profile.txt"); + + List commandLine = linkAction.getLinkCommandLineForTesting().getRawLinkArgv(); + assertThat(commandLine.toString()) + .containsMatch("-Wl,--symbol-ordering-file=.*/ld_profile.txt"); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + + String expectedCompilerFlag = "-fbasic-block-sections=list=.*/cc_profile.txt"; + assertThat(Joiner.on(" ").join(backendAction.getArguments())) + .containsMatch(expectedCompilerFlag); + String expectedBuildTypeFlag = "-DBUILD_PROPELLER_TYPE=\"full\""; + assertThat(Joiner.on(" ").join(backendAction.getArguments())) + .containsMatch(expectedBuildTypeFlag); + assertThat(ActionsTestUtil.baseArtifactNames(backendAction.getInputs())) + .contains("cc_profile.txt"); + + CppLinkAction indexAction = getIndexAction(backendAction); + assertThat(ActionsTestUtil.baseArtifactNames(indexAction.getInputs())) + .doesNotContain("ld_profile.txt"); + } + + @Test + public void testPropellerCcCompile() throws Exception { + createBuildFiles(); + + setupThinLTOCrosstool(CppRuleClasses.SUPPORTS_PIC, CppRuleClasses.AUTOFDO); + + useConfiguration( + "--propeller_optimize_absolute_cc_profile=/tmp/cc_profile.txt", + "--propeller_optimize_absolute_ld_profile=/tmp/ld_profile.txt", + "--compilation_mode=opt"); + Artifact binArtifact = getFilesToBuild(getCurrentTarget()).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + CppLinkAction indexAction = getIndexAction(backendAction); + CppCompileAction bitcodeAction = + (CppCompileAction) + getPredecessorByInputName(indexAction, "pkg/_objs/bin/binfile.indexing.o"); + assertThat(ActionsTestUtil.baseArtifactNames(bitcodeAction.getInputs())) + .doesNotContain("cc_profile.txt"); + assertThat(Joiner.on(" ").join(bitcodeAction.getArguments())) + .doesNotContainMatch("-fbasic-block-sections="); + } + + /** + * Check that the temporary opt-out from disabling Propeller profiles for ThinLTO compile actions + * works. + * + *

      TODO(b/182804945): Remove after making sure that the rollout of the new Propeller profile + * passing logic didn't break anything. + */ + @Test + public void testPropellerCcCompileWithPropellerOptimizeThinLtoCompileActions() throws Exception { + createBuildFiles(); + + setupThinLTOCrosstool(CppRuleClasses.SUPPORTS_PIC, CppRuleClasses.AUTOFDO); + + useConfiguration( + "--propeller_optimize_absolute_cc_profile=/tmp/cc_profile.txt", + "--propeller_optimize_absolute_ld_profile=/tmp/ld_profile.txt", + "--compilation_mode=opt", + "--features=propeller_optimize_thinlto_compile_actions", + "--features=use_lto_native_object_directory"); + Artifact binArtifact = getFilesToBuild(getCurrentTarget()).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + CppLinkAction indexAction = getIndexAction(backendAction); + assertThat(artifactsToStrings(indexAction.getInputs())) + .containsAtLeast( + "bin pkg/_objs/bin/binfile.indexing.o", "bin pkg/_objs/lib/libfile.indexing.o"); + + CppCompileAction bitcodeAction = + (CppCompileAction) + getPredecessorByInputName(indexAction, "pkg/_objs/bin/binfile.indexing.o"); + assertThat(ActionsTestUtil.baseArtifactNames(bitcodeAction.getInputs())) + .contains("cc_profile.txt"); + assertThat(Joiner.on(" ").join(bitcodeAction.getArguments())) + .containsMatch("-fbasic-block-sections=list=.*/cc_profile.txt"); + } + + @Test + public void testPropellerHostBuilds() throws Exception { + scratch.file( + "pkg/BUILD", + "package(features = ['thin_lto', 'use_lto_native_object_directory'])", + "", + "cc_binary(name = '" + targetName + "',", + " srcs = ['binfile.cc', ],", + " deps = [ ':lib' ], ", + " malloc = '//base:system_malloc')", + "cc_library(name = 'lib',", + " srcs = ['libfile.cc'],", + " hdrs = ['libfile.h'])", + "cc_binary(name = 'gen_lib',", + " srcs = ['gen_lib.cc'])", + "genrule(name = 'lib_genrule',", + " srcs = [],", + " outs = ['libfile.cc'],", + " cmd = '$(location gen_lib) > \"$@\"',", + " tools = [':gen_lib'])"); + + scratch.file("pkg/binfile.cc", "#include \"pkg/libfile.h\"", "int main() { return pkg(); }"); + scratch.file( + "pkg/gen_lib.cc", + "#include ", + "int main() { puts(\"int pkg() { return 42; }\"); }"); + scratch.file("pkg/libfile.h", "int pkg();"); + + setupThinLTOCrosstool(CppRuleClasses.SUPPORTS_PIC, CppRuleClasses.AUTOFDO); + + useConfiguration( + "--propeller_optimize_absolute_cc_profile=/tmp/cc_profile.txt", + "--propeller_optimize_absolute_ld_profile=/tmp/ld_profile.txt", + "--compilation_mode=opt"); + Artifact binArtifact = getFilesToBuild(getCurrentTarget()).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + CppLinkAction indexAction = getIndexAction(backendAction); + assertThat(artifactsToStrings(indexAction.getInputs())) + .contains("bin pkg/_objs/lib/libfile.indexing.o"); + + CppCompileAction bitcodeAction = + (CppCompileAction) + getPredecessorByInputName(indexAction, "pkg/_objs/lib/libfile.indexing.o"); + + Action genruleAction = getPredecessorByInputName(bitcodeAction, "pkg/libfile.cc"); + + CppLinkAction hostLinkAction = + (CppLinkAction) getPredecessorByInputName(genruleAction, "pkg/gen_lib"); + assertThat(ActionsTestUtil.baseArtifactNames(hostLinkAction.getInputs())) + .doesNotContain("ld_profile.txt"); + assertThat(hostLinkAction.getLinkCommandLineForTesting().getRawLinkArgv().toString()) + .doesNotContainMatch("-Wl,--symbol-ordering-file=.*/ld_profile.txt"); + + // The hostLinkAction inputs has a different root from the backendAction. + // Here we confirm that the correct root is on the path + String hostrootExecPath = hostLinkAction.getPrimaryOutput().getRoot().getExecPathString(); + LtoBackendAction hostBackendAction = + (LtoBackendAction) + getPredecessorByInputName( + hostLinkAction, + "pkg/gen_lib.lto-obj/" + hostrootExecPath + "/pkg/_objs/gen_lib/gen_lib.o"); + assertThat(ActionsTestUtil.baseArtifactNames(hostBackendAction.getInputs())) + .doesNotContain("cc_profile.txt"); + assertThat(Joiner.on(" ").join(hostBackendAction.getArguments())) + .doesNotContainMatch("-fbasic-block-sections"); + + CppLinkAction hostIndexAction = getIndexAction(hostBackendAction); + assertThat(hostIndexAction).isNotNull(); + assertThat(ActionsTestUtil.baseArtifactNames(hostIndexAction.getInputs())) + .doesNotContain("ld_profile.txt"); + assertThat(hostIndexAction.getLinkCommandLineForTesting().getRawLinkArgv().toString()) + .doesNotContainMatch("-Wl,--symbol-ordering-file=.*/ld_profile.txt"); + + CppCompileAction hostBitcodeAction = + (CppCompileAction) + getPredecessorByInputName(hostIndexAction, "pkg/_objs/gen_lib/gen_lib.indexing.o"); + assertThat(ActionsTestUtil.baseArtifactNames(hostBitcodeAction.getInputs())) + .doesNotContain("cc_profile.txt"); + assertThat(Joiner.on(" ").join(hostBitcodeAction.getArguments())) + .doesNotContainMatch("-fbasic-block-sections="); + } + + private void testPropellerOptimizeOption(boolean label) throws Exception { + createBuildFiles(); + + if (label) { + scratch.file( + "fdo/BUILD", + "propeller_optimize(name='test_propeller_optimize', cc_profile=':cc_profile.txt'," + + " ld_profile=':ld_profile.txt')"); + } else { + scratch.file( + "fdo/BUILD", + "propeller_optimize(name='test_propeller_optimize'," + + "absolute_cc_profile='/tmp/cc_profile.txt'," + + "absolute_ld_profile='/tmp/ld_profile.txt')"); + } + setupThinLTOCrosstool(CppRuleClasses.SUPPORTS_PIC, CppRuleClasses.AUTOFDO); + + useConfiguration( + "--propeller_optimize=//fdo:test_propeller_optimize", "--compilation_mode=opt"); + + Artifact binArtifact = getFilesToBuild(getCurrentTarget()).getSingleton(); + String rootExecPath = binArtifact.getRoot().getExecPathString(); + + CppLinkAction linkAction = (CppLinkAction) getGeneratingAction(binArtifact); + assertThat(linkAction.getOutputs()).containsExactly(binArtifact); + + List commandLine = linkAction.getLinkCommandLineForTesting().getRawLinkArgv(); + assertThat(commandLine.toString()) + .containsMatch("-Wl,--symbol-ordering-file=.*/ld_profile.txt"); + + LtoBackendAction backendAction = + (LtoBackendAction) + getPredecessorByInputName( + linkAction, "pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + + String expectedCompilerFlag = "-fbasic-block-sections=list=.*/cc_profile.txt"; + assertThat(Joiner.on(" ").join(backendAction.getArguments())) + .containsMatch(expectedCompilerFlag); + String expectedBuildTypeFlag = "-DBUILD_PROPELLER_TYPE=\"full\""; + assertThat(Joiner.on(" ").join(backendAction.getArguments())) + .containsMatch(expectedBuildTypeFlag); + assertThat(ActionsTestUtil.baseArtifactNames(backendAction.getInputs())) + .contains("cc_profile.txt"); + } + + @Test + public void testPropellerOptimizeOptionFromAbsolutePath() throws Exception { + testPropellerOptimizeOption(false); + } + + @Test + public void testPropellerOptimizeOptionFromLabel() throws Exception { + testPropellerOptimizeOption(true); + } + + private void testLLVMCachePrefetchBackendOption(String extraOption, boolean asLabel) + throws Exception { + createBuildFiles(); + if (asLabel) { + scratch.file( + "fdo/BUILD", "fdo_prefetch_hints(name='test_profile', profile=':prefetch.afdo')"); + } else { + scratch.file( + "fdo/BUILD", + "fdo_prefetch_hints(name='test_profile', absolute_path_profile='/tmp/prefetch.afdo')"); + } + + setupThinLTOCrosstool(CppRuleClasses.SUPPORTS_PIC, CppRuleClasses.AUTOFDO); + useConfiguration( + "--fdo_prefetch_hints=//fdo:test_profile", "--compilation_mode=opt", extraOption); + + String rootExecPath = getRootExecPath(); + LtoBackendAction backendAction = + getBackendAction("pkg/bin.lto-obj/" + rootExecPath + "/pkg/_objs/bin/binfile.o"); + + String expectedCompilerFlag = + "-prefetch-hints-file=" + + (asLabel ? ".*/prefetch.afdo" : "(blaze|bazel)-out/.*/fdo/.*/prefetch.afdo"); + assertThat(Joiner.on(" ").join(backendAction.getArguments())) + .containsMatch("-mllvm " + expectedCompilerFlag); + + assertThat(ActionsTestUtil.baseArtifactNames(backendAction.getInputs())) + .contains("prefetch.afdo"); + } + + @Test + public void testFdoCachePrefetchLLVMOptionsToBackendFromPath() throws Exception { + testLLVMCachePrefetchBackendOption("", false); + } + + @Test + public void testFdoCachePrefetchAndFdoLLVMOptionsToBackendFromPath() throws Exception { + testLLVMCachePrefetchBackendOption("--fdo_optimize=/profile.zip", false); + } + + @Test + public void testFdoCachePrefetchLLVMOptionsToBackendFromLabel() throws Exception { + testLLVMCachePrefetchBackendOption("", true); + } + + @Test + public void testFdoCachePrefetchAndFdoLLVMOptionsToBackendFromLabel() throws Exception { + testLLVMCachePrefetchBackendOption("--fdo_optimize=/profile.zip", true); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/rules/cpp/StarlarkCcCommonTest.java b/src/test/java/com/google/devtools/build/lib/rules/cpp/StarlarkCcCommonTest.java index 686edb7f82f2de..693d51b30730d9 100755 --- a/src/test/java/com/google/devtools/build/lib/rules/cpp/StarlarkCcCommonTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/cpp/StarlarkCcCommonTest.java @@ -7251,7 +7251,8 @@ public void testExpandedLtoAndFdoApiRaisesError() throws Exception { " fdo_context = toolchain.fdo_context()", " branch_fdo_profile = fdo_context.branch_fdo_profile()", " lto_backend_artifacts = cc_common.create_lto_backend_artifacts(ctx=ctx,", - " lto_output_root_prefix=ctx.label.package, bitcode_file=ctx.file.file,", + " lto_output_root_prefix=ctx.label.package, lto_obj_root_prefix=ctx.label.package,", + " bitcode_file=ctx.file.file,", " feature_configuration=feature_configuration, cc_toolchain=toolchain,", " fdo_context=fdo_context, use_pic=True,", " should_create_per_object_debug_info=False, argv=[])", diff --git a/src/test/java/com/google/devtools/build/lib/rules/objc/BUILD b/src/test/java/com/google/devtools/build/lib/rules/objc/BUILD index 716881324a8774..76666c6b13a512 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/objc/BUILD +++ b/src/test/java/com/google/devtools/build/lib/rules/objc/BUILD @@ -32,6 +32,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:config/build_configuration", "//src/main/java/com/google/devtools/build/lib/analysis:configured_target", "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/lib/collect/nestedset", "//src/main/java/com/google/devtools/build/lib/packages", "//src/main/java/com/google/devtools/build/lib/rules/apple", "//src/main/java/com/google/devtools/build/lib/rules/cpp", diff --git a/src/test/java/com/google/devtools/build/lib/rules/objc/BazelJ2ObjcLibraryTest.java b/src/test/java/com/google/devtools/build/lib/rules/objc/BazelJ2ObjcLibraryTest.java index 0440e6aadfc907..09206562d80758 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/objc/BazelJ2ObjcLibraryTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/objc/BazelJ2ObjcLibraryTest.java @@ -43,7 +43,10 @@ import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.RepositoryName; -import com.google.devtools.build.lib.packages.NativeAspectClass; +import com.google.devtools.build.lib.collect.nestedset.Depset; +import com.google.devtools.build.lib.packages.Provider; +import com.google.devtools.build.lib.packages.StarlarkProvider; +import com.google.devtools.build.lib.packages.StructImpl; import com.google.devtools.build.lib.packages.util.MockObjcSupport; import com.google.devtools.build.lib.rules.apple.ApplePlatform.PlatformType; import com.google.devtools.build.lib.rules.apple.AppleToolchain; @@ -74,8 +77,21 @@ */ @RunWith(JUnit4.class) public class BazelJ2ObjcLibraryTest extends J2ObjcLibraryTest { - protected NativeAspectClass getJ2ObjcAspect() { - return ruleClassProvider.getNativeAspectClass(J2ObjcAspect.NAME); + + private static final Provider.Key starlarkJ2objcMappingFileProviderKey = + new StarlarkProvider.Key( + Label.parseCanonicalUnchecked("@_builtins//:common/objc/providers.bzl"), + "J2ObjcMappingFileInfo"); + + private StructImpl getJ2ObjcMappingFileInfoFromTarget(ConfiguredTarget configuredTarget) + throws Exception { + return (StructImpl) configuredTarget.get(starlarkJ2objcMappingFileProviderKey); + } + + private ImmutableList getArtifacts(StructImpl j2ObjcMappingFileInfo, String attribute) + throws Exception { + Depset filesDepset = (Depset) j2ObjcMappingFileInfo.getValue(attribute); + return filesDepset.toList(Artifact.class); } /** @@ -244,9 +260,11 @@ public void testJ2ObjcHeaderMapExportedInJavaLibrary() throws Exception { ")"); ConfiguredTarget target = getJ2ObjCAspectConfiguredTarget("//java/com/google/transpile:dummy"); - J2ObjcMappingFileProvider provider = target.get(J2ObjcMappingFileProvider.PROVIDER); + StructImpl j2ObjcMappingFileInfo = getJ2ObjcMappingFileInfoFromTarget(target); + ImmutableList headerMappingFilesList = + getArtifacts(j2ObjcMappingFileInfo, "header_mapping_files"); - assertThat(provider.getHeaderMappingFiles().getSingleton().getRootRelativePath().toString()) + assertThat(headerMappingFilesList.get(0).getRootRelativePath().toString()) .isEqualTo("java/com/google/transpile/dummy.mapping.j2objc"); } @@ -266,9 +284,11 @@ public void testDepsJ2ObjcHeaderMapExportedInJavaLibraryWithNoSourceFile() throw ")"); ConfiguredTarget target = getJ2ObjCAspectConfiguredTarget("//java/com/google/transpile:dummy"); - J2ObjcMappingFileProvider provider = target.get(J2ObjcMappingFileProvider.PROVIDER); + StructImpl j2ObjcMappingFileInfo = getJ2ObjcMappingFileInfoFromTarget(target); + ImmutableList headerMappingFilesList = + getArtifacts(j2ObjcMappingFileInfo, "header_mapping_files"); - assertThat(provider.getHeaderMappingFiles().getSingleton().getRootRelativePath().toString()) + assertThat(headerMappingFilesList.get(0).getRootRelativePath().toString()) .isEqualTo("java/com/google/dep/dep.mapping.j2objc"); } @@ -296,15 +316,13 @@ public void testJ2ObjcProtoClassMappingFilesExportedInJavaLibrary() throws Excep ConfiguredTarget target = getJ2ObjCAspectConfiguredTarget( "//java/com/google/dummy/test/proto:test"); - J2ObjcMappingFileProvider provider = target.get(J2ObjcMappingFileProvider.PROVIDER); - Artifact classMappingFile = - getGenfilesArtifact( - "test.clsmap.properties", - getConfiguredTarget( - "//java/com/google/dummy/test/proto:test_proto", getAppleCrosstoolConfiguration()), - getJ2ObjcAspect()); - - assertThat(provider.getClassMappingFiles().toList()).containsExactly(classMappingFile); + StructImpl j2ObjcMappingFileInfo = getJ2ObjcMappingFileInfoFromTarget(target); + ImmutableList classMappingFilesList = + getArtifacts(j2ObjcMappingFileInfo, "class_mapping_files"); + + assertThat(classMappingFilesList.get(0).getExecPathString()) + .containsMatch( + "/applebin_macos-darwin_x86_64-fastbuild-ST-[^/]*/bin/java/com/google/dummy/test/proto/test.clsmap.properties"); } @Test @@ -327,19 +345,20 @@ public void testJavaProtoLibraryWithProtoLibrary() throws Exception { ")"); ConfiguredTarget target = getJ2ObjCAspectConfiguredTarget("//x:test"); - ConfiguredTarget test = getConfiguredTarget("//x:test_proto", getAppleCrosstoolConfiguration()); - J2ObjcMappingFileProvider provider = target.get(J2ObjcMappingFileProvider.PROVIDER); - Artifact classMappingFile = - getGenfilesArtifact("test.clsmap.properties", test, getJ2ObjcAspect()); - assertThat(provider.getClassMappingFiles().toList()).containsExactly(classMappingFile); + StructImpl j2ObjcMappingFileInfo = getJ2ObjcMappingFileInfoFromTarget(target); + ImmutableList classMappingFilesList = + getArtifacts(j2ObjcMappingFileInfo, "class_mapping_files"); + assertThat(classMappingFilesList.get(0).getExecPathString()) + .containsMatch( + "/applebin_macos-darwin_x86_64-fastbuild-ST-[^/]*/bin/x/test.clsmap.properties"); ObjcProvider objcProvider = target.get(ObjcProvider.STARLARK_CONSTRUCTOR); CcCompilationContext ccCompilationContext = target.get(CcInfo.PROVIDER).getCcCompilationContext(); - Artifact headerFile = getGenfilesArtifact("test.j2objc.pb.h", test, getJ2ObjcAspect()); - Artifact sourceFile = getGenfilesArtifact("test.j2objc.pb.m", test, getJ2ObjcAspect()); - assertThat(ccCompilationContext.getDeclaredIncludeSrcs().toList()).contains(headerFile); - assertThat(objcProvider.get(ObjcProvider.SOURCE).toList()).contains(sourceFile); + assertThat(ccCompilationContext.getDeclaredIncludeSrcs().toList().toString()) + .containsMatch("/applebin_macos-darwin_x86_64-fastbuild-ST-[^/]*/bin]x/test.j2objc.pb.h"); + assertThat(objcProvider.get(ObjcProvider.SOURCE).toList().toString()) + .containsMatch("/applebin_macos-darwin_x86_64-fastbuild-ST-[^/]*/bin]x/test.j2objc.pb.m,"); } @Test @@ -374,25 +393,25 @@ public void testJavaProtoLibraryWithProtoLibrary_external() throws Exception { " deps = ['@bla//foo:test_java_proto'])"); ConfiguredTarget target = getJ2ObjCAspectConfiguredTarget("//x:test"); - ConfiguredTarget test = - getConfiguredTarget("@bla//foo:test_proto", getAppleCrosstoolConfiguration()); - J2ObjcMappingFileProvider provider = target.get(J2ObjcMappingFileProvider.PROVIDER); + StructImpl j2ObjcMappingFileInfo = getJ2ObjcMappingFileInfoFromTarget(target); + ImmutableList classMappingFilesList = + getArtifacts(j2ObjcMappingFileInfo, "class_mapping_files"); - Artifact classMappingFile = - getGenfilesArtifact("../external/bla/foo/test.clsmap.properties", test, getJ2ObjcAspect()); - assertThat(provider.getClassMappingFiles().toList()).containsExactly(classMappingFile); + assertThat(classMappingFilesList.get(0).getExecPathString()) + .containsMatch( + "/applebin_macos-darwin_x86_64-fastbuild-ST-[^/]*/bin/external/bla/foo/test.clsmap.properties"); ObjcProvider objcProvider = target.get(ObjcProvider.STARLARK_CONSTRUCTOR); CcCompilationContext ccCompilationContext = target.get(CcInfo.PROVIDER).getCcCompilationContext(); - Artifact headerFile = - getGenfilesArtifact("../external/bla/foo/test.j2objc.pb.h", test, getJ2ObjcAspect()); - Artifact sourceFile = - getGenfilesArtifact("../external/bla/foo/test.j2objc.pb.m", test, getJ2ObjcAspect()); - assertThat(ccCompilationContext.getDeclaredIncludeSrcs().toList()).contains(headerFile); - assertThat(objcProvider.get(ObjcProvider.SOURCE).toList()).contains(sourceFile); + assertThat(ccCompilationContext.getDeclaredIncludeSrcs().toList().toString()) + .containsMatch( + "/applebin_macos-darwin_x86_64-fastbuild-ST-[^/]*/bin]external/bla/foo/test.j2objc.pb.h"); + assertThat(objcProvider.get(ObjcProvider.SOURCE).toList().toString()) + .containsMatch( + "/applebin_macos-darwin_x86_64-fastbuild-ST-[^/]*/bin]external/bla/foo/test.j2objc.pb.m"); assertThat(ccCompilationContext.getIncludeDirs()) .contains( getConfiguration(target) @@ -411,11 +430,15 @@ public void testJ2ObjcInfoExportedInJavaImport() throws Exception { ")"); ConfiguredTarget target = getJ2ObjCAspectConfiguredTarget("//java/com/google/transpile:dummy"); - J2ObjcMappingFileProvider provider = target.get(J2ObjcMappingFileProvider.PROVIDER); + StructImpl j2ObjcMappingFileInfo = getJ2ObjcMappingFileInfoFromTarget(target); + ImmutableList headerMappingFilesList = + getArtifacts(j2ObjcMappingFileInfo, "header_mapping_files"); + ImmutableList classMappingFilesList = + getArtifacts(j2ObjcMappingFileInfo, "class_mapping_files"); - assertThat(provider.getHeaderMappingFiles().getSingleton().getRootRelativePath().toString()) + assertThat(headerMappingFilesList.get(0).getRootRelativePath().toString()) .isEqualTo("java/com/google/transpile/dummy.mapping.j2objc"); - assertThat(provider.getClassMappingFiles().toList()).isEmpty(); + assertThat(classMappingFilesList).isEmpty(); } protected void checkObjcArchiveAndLinkActions( @@ -443,7 +466,8 @@ public void testMissingEntryClassesError() throws Exception { checkError( "java/com/google/dummy", "transpile", - J2ObjcLibrary.NO_ENTRY_CLASS_ERROR_MSG, + "Entry classes must be specified when flag --compilation_mode=opt is on in order to perform" + + " J2ObjC dead code stripping.", "j2objc_library(", " name = 'transpile',", " deps = ['//java/com/google/dummy/test:test'],", @@ -581,12 +605,14 @@ public void testJ2ObjcHeaderMappingAction() throws Exception { ConfiguredTarget target = getJ2ObjCAspectConfiguredTarget( "//java/com/google/transpile:lib1"); - J2ObjcMappingFileProvider mappingFileProvider = target.get(J2ObjcMappingFileProvider.PROVIDER); - assertThat(baseArtifactNames(mappingFileProvider.getHeaderMappingFiles())) + StructImpl j2ObjcMappingFileInfo = getJ2ObjcMappingFileInfoFromTarget(target); + ImmutableList headerMappingFilesList = + getArtifacts(j2ObjcMappingFileInfo, "header_mapping_files"); + assertThat(baseArtifactNames(headerMappingFilesList)) .containsExactly("lib1.mapping.j2objc", "lib2.mapping.j2objc"); - Artifact mappingFile = getFirstArtifactEndingWith( - mappingFileProvider.getHeaderMappingFiles(), "lib1.mapping.j2objc"); + Artifact mappingFile = + getFirstArtifactEndingWith(headerMappingFilesList, "lib1.mapping.j2objc"); SpawnAction headerMappingAction = (SpawnAction) getGeneratingAction(mappingFile); String execPath = getRuleContext(target).getBinFragment() + "/"; assertThat(baseArtifactNames(headerMappingAction.getInputs())) @@ -1261,8 +1287,6 @@ public void testJ2ObjcDeadCodeRemovalActionWithOptFlag() throws Exception { execPath + TestConstants.TOOLS_REPOSITORY_PATH_PREFIX + "tools/objc/dummy/libdummy_lib.a", - "--xcrunwrapper", - removeConfigFragment(MOCK_XCRUNWRAPPER_EXECUTABLE_PATH), "--dependency_mapping_files", removeConfigFragment(dependencyMappingFile.getExecPathString()), "--header_mapping_files", diff --git a/src/test/java/com/google/devtools/build/lib/rules/objc/J2ObjcSourceTest.java b/src/test/java/com/google/devtools/build/lib/rules/objc/J2ObjcSourceTest.java deleted file mode 100644 index fc7b6c1a1ab0db..00000000000000 --- a/src/test/java/com/google/devtools/build/lib/rules/objc/J2ObjcSourceTest.java +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2017 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.rules.objc; - -import com.google.common.collect.ImmutableList; -import com.google.common.testing.EqualsTester; -import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.actions.ArtifactRoot; -import com.google.devtools.build.lib.actions.ArtifactRoot.RootType; -import com.google.devtools.build.lib.actions.util.ActionsTestUtil; -import com.google.devtools.build.lib.cmdline.Label; -import com.google.devtools.build.lib.testutil.Scratch; -import com.google.devtools.build.lib.vfs.Path; -import com.google.devtools.build.lib.vfs.PathFragment; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** - * Unit test for {@link J2ObjcSource}. - */ -@RunWith(JUnit4.class) -public class J2ObjcSourceTest { - private ArtifactRoot rootDir; - - @Before - public final void setRootDir() throws Exception { - Scratch scratch = new Scratch(); - Path execRoot = scratch.getFileSystem().getPath("/exec"); - String outSegment = "root"; - execRoot.getChild(outSegment).createDirectoryAndParents(); - rootDir = ArtifactRoot.asDerivedRoot(execRoot, RootType.Output, outSegment); - } - - @Test - public void testEqualsAndHashCode() throws Exception { - new EqualsTester() - .addEqualityGroup( - getJ2ObjcSource("//a/b:c", "sourceA", J2ObjcSource.SourceType.JAVA, false), - getJ2ObjcSource("//a/b:c", "sourceA", J2ObjcSource.SourceType.JAVA, false)) - .addEqualityGroup( - getJ2ObjcSource("//a/b:d", "sourceA", J2ObjcSource.SourceType.JAVA, false), - getJ2ObjcSource("//a/b:d", "sourceA", J2ObjcSource.SourceType.JAVA, false)) - .addEqualityGroup( - getJ2ObjcSource("//a/b:d", "sourceC", J2ObjcSource.SourceType.JAVA, false), - getJ2ObjcSource("//a/b:d", "sourceC", J2ObjcSource.SourceType.JAVA, false)) - .addEqualityGroup( - getJ2ObjcSource("//a/b:d", "sourceC", J2ObjcSource.SourceType.PROTO, false), - getJ2ObjcSource("//a/b:d", "sourceC", J2ObjcSource.SourceType.PROTO, false)) - .addEqualityGroup( - getJ2ObjcSource("//a/b:c", "sourceA", J2ObjcSource.SourceType.JAVA, true), - getJ2ObjcSource("//a/b:c", "sourceA", J2ObjcSource.SourceType.JAVA, true)) - .addEqualityGroup( - getJ2ObjcSource("//a/b:d", "sourceA", J2ObjcSource.SourceType.JAVA, true), - getJ2ObjcSource("//a/b:d", "sourceA", J2ObjcSource.SourceType.JAVA, true)) - .addEqualityGroup( - getJ2ObjcSource("//a/b:d", "sourceC", J2ObjcSource.SourceType.JAVA, true), - getJ2ObjcSource("//a/b:d", "sourceC", J2ObjcSource.SourceType.JAVA, true)) - .addEqualityGroup( - getJ2ObjcSource("//a/b:d", "sourceC", J2ObjcSource.SourceType.PROTO, true), - getJ2ObjcSource("//a/b:d", "sourceC", J2ObjcSource.SourceType.PROTO, true)) - .testEquals(); - } - - private J2ObjcSource getJ2ObjcSource( - String label, String fileName, J2ObjcSource.SourceType sourceType, boolean compileWithARC) - throws Exception { - Label ruleLabel = Label.parseCanonical(label); - PathFragment path = ruleLabel.toPathFragment(); - return new J2ObjcSource( - ruleLabel, - ImmutableList.of(getArtifactForTest(path.getRelative(fileName + ".m").toString())), - ImmutableList.of(getArtifactForTest(path.getRelative(fileName + ".h").toString())), - path, - sourceType, - ImmutableList.of(path), - compileWithARC); - } - - private Artifact getArtifactForTest(String path) throws Exception { - return ActionsTestUtil.createArtifact(rootDir, path); - } -} diff --git a/src/test/java/com/google/devtools/build/lib/rules/objc/ObjcLibraryTest.java b/src/test/java/com/google/devtools/build/lib/rules/objc/ObjcLibraryTest.java index 33a237c2fb6b2b..4c7e4d3f552c04 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/objc/ObjcLibraryTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/objc/ObjcLibraryTest.java @@ -49,10 +49,14 @@ import com.google.devtools.build.lib.analysis.test.InstrumentedFilesInfo; import com.google.devtools.build.lib.analysis.util.AnalysisMock; import com.google.devtools.build.lib.analysis.util.ScratchAttributeWriter; +import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.Provider; +import com.google.devtools.build.lib.packages.StarlarkProvider; +import com.google.devtools.build.lib.packages.StructImpl; import com.google.devtools.build.lib.packages.util.MockObjcSupport; import com.google.devtools.build.lib.rules.apple.AppleToolchain; import com.google.devtools.build.lib.rules.cpp.CcCompilationContext; @@ -1586,11 +1590,21 @@ public void testApplePlatformEnvForCcLibraryDep() throws Exception { assertAppleSdkVersionEnv(action.getIncompleteEnvironmentForTesting()); } + private StructImpl getJ2ObjcInfoFromTarget(ConfiguredTarget configuredTarget, String providerName) + throws Exception { + Provider.Key key = + new StarlarkProvider.Key( + Label.parseCanonical("@_builtins//:common/objc/providers.bzl"), providerName); + return (StructImpl) configuredTarget.get(key); + } + @Test public void testExportsJ2ObjcProviders() throws Exception { ConfiguredTarget lib = createLibraryTargetWriter("//a:lib").write(); - assertThat(lib.get(J2ObjcEntryClassProvider.PROVIDER)).isNotNull(); - assertThat(lib.get(J2ObjcMappingFileProvider.PROVIDER)).isNotNull(); + StructImpl j2ObjcEntryClassInfo = getJ2ObjcInfoFromTarget(lib, "J2ObjcEntryClassInfo"); + StructImpl j2ObjcMappingFileInfo = getJ2ObjcInfoFromTarget(lib, "J2ObjcMappingFileInfo"); + assertThat(j2ObjcEntryClassInfo).isNotNull(); + assertThat(j2ObjcMappingFileInfo).isNotNull(); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java index 91c261a465f263..8a951990f9d899 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java @@ -36,6 +36,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleValue; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; +import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsUtil; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; @@ -273,7 +274,7 @@ public void setupDelegator() throws Exception { RepositoryDelegatorFunction.RESOLVED_FILE_FOR_VERIFICATION.set(differencer, Optional.empty()); ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, false); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); + YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( differencer, CheckDirectDepsMode.WARNING); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( diff --git a/src/test/java/com/google/devtools/build/lib/rules/starlarkdocextract/StarlarkDocExtractTest.java b/src/test/java/com/google/devtools/build/lib/rules/starlarkdocextract/StarlarkDocExtractTest.java index eed32f0a6922c4..baacd649254b22 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/starlarkdocextract/StarlarkDocExtractTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/starlarkdocextract/StarlarkDocExtractTest.java @@ -31,6 +31,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.BzlmodTestUtil; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; +import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsUtil; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; @@ -77,8 +78,7 @@ protected ImmutableList extraPrecomputedValues() { ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl())), PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false), PrecomputedValue.injected(ModuleFileFunction.MODULE_OVERRIDES, ImmutableMap.of()), - PrecomputedValue.injected( - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), + PrecomputedValue.injected(YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), PrecomputedValue.injected( BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/BzlLoadFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/BzlLoadFunctionTest.java index f247b5d7fc3000..89d1ad224dfc9f 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/BzlLoadFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/BzlLoadFunctionTest.java @@ -26,6 +26,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; +import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsUtil; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; @@ -109,8 +110,7 @@ protected ImmutableList extraPrecomputedValues() { ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl())), PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false), PrecomputedValue.injected(ModuleFileFunction.MODULE_OVERRIDES, ImmutableMap.of()), - PrecomputedValue.injected( - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), + PrecomputedValue.injected(YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), PrecomputedValue.injected( BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKeyTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKeyTest.java index c391e943715024..c44c5b4b2d3e80 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKeyTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetKeyTest.java @@ -13,24 +13,106 @@ // limitations under the License. package com.google.devtools.build.lib.skyframe; +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.config.BuildOptions; import com.google.devtools.build.lib.analysis.config.BuildOptions.MapBackedChecksumCache; import com.google.devtools.build.lib.analysis.config.BuildOptions.OptionsChecksumCache; +import com.google.devtools.build.lib.analysis.config.CoreOptions; import com.google.devtools.build.lib.analysis.util.BuildViewTestCase; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.skyframe.serialization.testutils.SerializationTester; +import com.google.testing.junit.testparameterinjector.TestParameter; import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.Test; import org.junit.runner.RunWith; @RunWith(TestParameterInjector.class) public final class ConfiguredTargetKeyTest extends BuildViewTestCase { + private static final AtomicInteger nextId = new AtomicInteger(); + + @Test + public void testDelegation( + @TestParameter boolean useNullConfig, @TestParameter boolean isToolchainKey) { + var baseKey = createKey(useNullConfig, isToolchainKey); + + assertThat(baseKey.isProxy()).isFalse(); + assertThat(baseKey.toKey()).isSameInstanceAs(baseKey); + + BuildConfigurationKey newConfigurationKey = getNewUniqueConfigurationKey(); + var delegatingKey = + ConfiguredTargetKey.builder() + .setDelegate(baseKey) + .setConfigurationKey(newConfigurationKey) + .build(); + assertThat(delegatingKey.isProxy()).isTrue(); + assertThat(delegatingKey.toKey()).isSameInstanceAs(baseKey); + assertThat(delegatingKey.getLabel()).isSameInstanceAs(baseKey.getLabel()); + assertThat(delegatingKey.getConfigurationKey()).isSameInstanceAs(newConfigurationKey); + assertThat(delegatingKey.getExecutionPlatformLabel()) + .isSameInstanceAs(baseKey.getExecutionPlatformLabel()); + + // Building a key with the same parameters as the delegating key returns the delegating key. + var similarKey = + ConfiguredTargetKey.builder() + .setLabel(delegatingKey.getLabel()) + .setConfigurationKey(delegatingKey.getConfigurationKey()) + .setExecutionPlatformLabel(delegatingKey.getExecutionPlatformLabel()) + .build(); + assertThat(similarKey).isSameInstanceAs(delegatingKey); + } + + @Test + public void existingKey_inhibitsDelegation( + @TestParameter boolean useNullConfig, @TestParameter boolean isToolchainKey) { + var baseKey = createKey(useNullConfig, isToolchainKey); + + BuildConfigurationKey newConfigurationKey = getNewUniqueConfigurationKey(); + + var existingKey = + ConfiguredTargetKey.builder() + .setLabel(baseKey.getLabel()) + .setConfigurationKey(newConfigurationKey) + .setExecutionPlatformLabel(baseKey.getExecutionPlatformLabel()) + .build(); + + var delegatingKey = + ConfiguredTargetKey.builder() + .setDelegate(baseKey) + .setConfigurationKey(newConfigurationKey) + .build(); + + assertThat(delegatingKey).isSameInstanceAs(existingKey); + } + @Test public void testCodec() throws Exception { var nullConfigKey = createKey(/* useNullConfig= */ true, /* isToolchainKey= */ false); var keyWithConfig = createKey(/* useNullConfig= */ false, /* isToolchainKey= */ false); var toolchainKey = createKey(/* useNullConfig= */ false, /* isToolchainKey= */ true); - new SerializationTester(nullConfigKey, keyWithConfig, toolchainKey) + var delegatingToNullConfig = + ConfiguredTargetKey.builder() + .setDelegate(nullConfigKey) + .setConfigurationKey(targetConfigKey) + .build(); + var delegatingToKeyWithConfig = + ConfiguredTargetKey.builder().setDelegate(keyWithConfig).build(); + var delegatingToToolchainKey = + ConfiguredTargetKey.builder() + .setDelegate(toolchainKey) + .setConfigurationKey(getNewUniqueConfigurationKey()) + .build(); + + new SerializationTester( + nullConfigKey, + keyWithConfig, + toolchainKey, + delegatingToNullConfig, + delegatingToKeyWithConfig, + delegatingToToolchainKey) .addDependency(OptionsChecksumCache.class, new MapBackedChecksumCache()) .runTests(); } @@ -45,4 +127,13 @@ private ConfiguredTargetKey createKey(boolean useNullConfig, boolean isToolchain } return key.build(); } + + private BuildConfigurationKey getNewUniqueConfigurationKey() { + BuildOptions newOptions = targetConfigKey.getOptions().clone(); + var coreOptions = newOptions.get(CoreOptions.class); + coreOptions.affectedByStarlarkTransition = + ImmutableList.of("//fake:id" + nextId.getAndIncrement()); + assertThat(newOptions.checksum()).isNotEqualTo(targetConfigKey.getOptions().checksum()); + return BuildConfigurationKey.withoutPlatformMapping(newOptions); + } } diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestorsTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestorsTest.java index 4b5086afec1d10..138dfa8acba290 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestorsTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestorsTest.java @@ -22,28 +22,18 @@ import static com.google.devtools.build.lib.testing.common.DirectoryListingHelper.file; import static com.google.devtools.build.lib.testing.common.DirectoryListingHelper.symlink; import static org.junit.Assert.assertThrows; -import static org.junit.Assert.fail; -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.common.collect.Streams; import com.google.devtools.build.lib.actions.FileStateValue; import com.google.devtools.build.lib.server.FailureDetails.DiffAwareness.Code; -import com.google.devtools.build.lib.testutil.Scratch; +import com.google.devtools.build.lib.skyframe.DirtinessCheckerUtils.FileDirtinessChecker; import com.google.devtools.build.lib.util.AbruptExitException; -import com.google.devtools.build.lib.vfs.DelegateFileSystem; -import com.google.devtools.build.lib.vfs.Dirent; import com.google.devtools.build.lib.vfs.FileStateKey; -import com.google.devtools.build.lib.vfs.FileStatus; -import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; -import com.google.devtools.build.lib.vfs.Root; -import com.google.devtools.build.lib.vfs.RootedPath; -import com.google.devtools.build.lib.vfs.SyscallCache; import com.google.devtools.build.skyframe.Differencer.DiffWithDelta.Delta; import com.google.devtools.build.skyframe.ImmutableDiff; import com.google.devtools.build.skyframe.SkyKey; @@ -51,73 +41,33 @@ import com.google.testing.junit.testparameterinjector.TestParameter; import com.google.testing.junit.testparameterinjector.TestParameterInjector; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import javax.annotation.Nullable; -import org.junit.After; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; /** Unit tests for {@link FileSystemValueCheckerInferringAncestors}. */ @RunWith(TestParameterInjector.class) -public final class FileSystemValueCheckerInferringAncestorsTest { +public final class FileSystemValueCheckerInferringAncestorsTest + extends FileSystemValueCheckerInferringAncestorsTestBase { private static final Delta DIRECTORY_FILE_STATE_NODE_DELTA = Delta.justNew(DIRECTORY_FILE_STATE_NODE); private static final Delta NONEXISTENT_FILE_STATE_NODE_DELTA = Delta.justNew(NONEXISTENT_FILE_STATE_NODE); - private final Scratch scratch = new Scratch(); - private final List statedPaths = new ArrayList<>(); - private final DefaultSyscallCache syscallCache = DefaultSyscallCache.newBuilder().build(); - private Root root; - private Root untrackedRoot; - private Exception throwOnStat; + private final SkyValueDirtinessChecker skyValueDirtinessChecker = new FileDirtinessChecker(); @TestParameter({"1", "16"}) private int fsvcThreads; - @Before - public void createRoot() throws IOException { - Path srcRootPath = scratch.dir("/src"); - PathFragment srcRoot = srcRootPath.asFragment(); - FileSystem trackingFileSystem = - new DelegateFileSystem(scratch.getFileSystem()) { - @Nullable - @Override - public synchronized FileStatus statIfFound(PathFragment path, boolean followSymlinks) - throws IOException { - if (throwOnStat != null) { - Exception toThrow = throwOnStat; - throwOnStat = null; - Throwables.propagateIfPossible(toThrow, IOException.class); - fail("Unexpected exception type"); - } - statedPaths.add(path.relativeTo(srcRoot).toString()); - return super.statIfFound(path, followSymlinks); - } - }; - root = Root.fromPath(trackingFileSystem.getPath(srcRoot)); - scratch.setWorkingDir("/src"); - untrackedRoot = Root.fromPath(srcRootPath); - } - - @After - public void checkExceptionThrown() { - assertThat(throwOnStat).isNull(); - syscallCache.clear(); - } - @Test public void getDiffWithInferredAncestors_unknownFileChanged_returnsFileAndDirs() throws Exception { ImmutableDiff diff = FileSystemValueCheckerInferringAncestors.getDiffWithInferredAncestors( - /*tsgm=*/ null, - /*graphValues=*/ ImmutableMap.of(), - /*graphDoneValues=*/ ImmutableMap.of(), - /*modifiedKeys=*/ ImmutableSet.of(fileStateValueKey("foo/file")), + /* tsgm= */ null, + inMemoryGraph, + /* modifiedKeys= */ ImmutableSet.of(fileStateValueKey("foo/file")), fsvcThreads, - syscallCache); + syscallCache, + skyValueDirtinessChecker); assertThat(diff.changedKeysWithoutNewValues()) .containsExactly( @@ -136,15 +86,16 @@ public void getDiffWithInferredAncestors_fileModified_returnsFileWithValues() th FileStateKey key = fileStateValueKey("file"); FileStateValue value = fileStateValue("file"); scratch.overwriteFile("file", "there"); + addDoneNodesAndThenMarkChanged(ImmutableMap.of(key, value)); ImmutableDiff diff = FileSystemValueCheckerInferringAncestors.getDiffWithInferredAncestors( - /*tsgm=*/ null, - /*graphValues=*/ ImmutableMap.of(fileStateValueKey("file"), value), - /*graphDoneValues=*/ ImmutableMap.of(), - /*modifiedKeys=*/ ImmutableSet.of(key), + /* tsgm= */ null, + inMemoryGraph, + /* modifiedKeys= */ ImmutableSet.of(key), fsvcThreads, - syscallCache); + syscallCache, + skyValueDirtinessChecker); Delta newValue = fileStateValueDelta("file"); assertThat(diff.changedKeysWithNewValues()).containsExactly(key, newValue); @@ -156,16 +107,18 @@ public void getDiffWithInferredAncestors_fileModified_returnsFileWithValues() th public void getDiffWithInferredAncestors_fileAdded_returnsFileAndDirListing() throws Exception { scratch.file("file"); FileStateKey key = fileStateValueKey("file"); + addDoneNodesAndThenMarkChanged( + ImmutableMap.of( + key, NONEXISTENT_FILE_STATE_NODE, fileStateValueKey(""), fileStateValue(""))); ImmutableDiff diff = FileSystemValueCheckerInferringAncestors.getDiffWithInferredAncestors( - /*tsgm=*/ null, - /*graphValues=*/ ImmutableMap.of( - key, NONEXISTENT_FILE_STATE_NODE, fileStateValueKey(""), fileStateValue("")), - /*graphDoneValues=*/ ImmutableMap.of(), - /*modifiedKeys=*/ ImmutableSet.of(key), + /* tsgm= */ null, + inMemoryGraph, + /* modifiedKeys= */ ImmutableSet.of(key), fsvcThreads, - syscallCache); + syscallCache, + skyValueDirtinessChecker); Delta delta = fileStateValueDelta("file"); assertThat(diff.changedKeysWithNewValues()).containsExactly(key, delta); @@ -179,23 +132,25 @@ public void getDiffWithInferredAncestors_fileWithDirsAdded_returnsFileAndInjects throws Exception { scratch.file("a/b/file"); FileStateKey fileKey = fileStateValueKey("a/b/file"); + addDoneNodesAndThenMarkChanged( + ImmutableMap.of( + fileStateValueKey(""), + fileStateValue(""), + fileStateValueKey("a"), + NONEXISTENT_FILE_STATE_NODE, + fileStateValueKey("a/b"), + NONEXISTENT_FILE_STATE_NODE, + fileKey, + NONEXISTENT_FILE_STATE_NODE)); ImmutableDiff diff = FileSystemValueCheckerInferringAncestors.getDiffWithInferredAncestors( - /*tsgm=*/ null, - /*graphValues=*/ ImmutableMap.of( - fileStateValueKey(""), - fileStateValue(""), - fileStateValueKey("a"), - NONEXISTENT_FILE_STATE_NODE, - fileStateValueKey("a/b"), - NONEXISTENT_FILE_STATE_NODE, - fileKey, - NONEXISTENT_FILE_STATE_NODE), - /*graphDoneValues=*/ ImmutableMap.of(), - /*modifiedKeys=*/ ImmutableSet.of(fileKey), + /* tsgm= */ null, + inMemoryGraph, + /* modifiedKeys= */ ImmutableSet.of(fileKey), fsvcThreads, - syscallCache); + syscallCache, + skyValueDirtinessChecker); Delta delta = fileStateValueDelta("a/b/file"); assertThat(diff.changedKeysWithNewValues()) @@ -219,23 +174,25 @@ public void getDiffWithInferredAncestors_addedFileWithReportedDirs_returnsFileAn throws Exception { scratch.file("a/b/file"); FileStateKey fileKey = fileStateValueKey("a/b/file"); + addDoneNodesAndThenMarkChanged( + ImmutableMap.of( + fileStateValueKey(""), + fileStateValue(""), + fileStateValueKey("a"), + NONEXISTENT_FILE_STATE_NODE, + fileStateValueKey("a/b"), + NONEXISTENT_FILE_STATE_NODE, + fileKey, + NONEXISTENT_FILE_STATE_NODE)); ImmutableDiff diff = FileSystemValueCheckerInferringAncestors.getDiffWithInferredAncestors( - /*tsgm=*/ null, - /*graphValues=*/ ImmutableMap.of( - fileStateValueKey(""), - fileStateValue(""), - fileStateValueKey("a"), - NONEXISTENT_FILE_STATE_NODE, - fileStateValueKey("a/b"), - NONEXISTENT_FILE_STATE_NODE, - fileKey, - NONEXISTENT_FILE_STATE_NODE), - /*graphDoneValues=*/ ImmutableMap.of(), - /*modifiedKeys=*/ ImmutableSet.of(fileKey, fileStateValueKey("a")), + /* tsgm= */ null, + inMemoryGraph, + /* modifiedKeys= */ ImmutableSet.of(fileKey, fileStateValueKey("a")), fsvcThreads, - syscallCache); + syscallCache, + skyValueDirtinessChecker); Delta newState = fileStateValueDelta("a/b/file"); assertThat(diff.changedKeysWithNewValues()) @@ -262,19 +219,21 @@ public void getDiffWithInferredAncestors_addedFileWithReportedDirs_returnsFileAn public void getDiffWithInferredAncestors_fileWithUnknownDirsAdded_returnsFileAndDirs() throws Exception { scratch.file("a/b/c/d"); + addDoneNodesAndThenMarkChanged( + ImmutableMap.of( + fileStateValueKey(""), + fileStateValue(""), + fileStateValueKey("a/b/c/d"), + NONEXISTENT_FILE_STATE_NODE)); ImmutableDiff diff = FileSystemValueCheckerInferringAncestors.getDiffWithInferredAncestors( - /*tsgm=*/ null, - /*graphValues=*/ ImmutableMap.of( - fileStateValueKey(""), - fileStateValue(""), - fileStateValueKey("a/b/c/d"), - NONEXISTENT_FILE_STATE_NODE), - /*graphDoneValues=*/ ImmutableMap.of(), - /*modifiedKeys=*/ ImmutableSet.of(fileStateValueKey("a/b/c/d")), + /* tsgm= */ null, + inMemoryGraph, + /* modifiedKeys= */ ImmutableSet.of(fileStateValueKey("a/b/c/d")), fsvcThreads, - syscallCache); + syscallCache, + skyValueDirtinessChecker); assertThat(diff.changedKeysWithoutNewValues()) .containsExactly( @@ -296,16 +255,18 @@ public void getDiffWithInferredAncestors_addEmptyDir_returnsDirAndParentListing( scratch.dir("dir"); FileStateKey key = fileStateValueKey("dir"); Delta delta = fileStateValueDelta("dir"); + addDoneNodesAndThenMarkChanged( + ImmutableMap.of( + key, NONEXISTENT_FILE_STATE_NODE, fileStateValueKey(""), fileStateValue(""))); ImmutableDiff diff = FileSystemValueCheckerInferringAncestors.getDiffWithInferredAncestors( - /*tsgm=*/ null, - /*graphValues=*/ ImmutableMap.of( - key, NONEXISTENT_FILE_STATE_NODE, fileStateValueKey(""), fileStateValue("")), - /*graphDoneValues=*/ ImmutableMap.of(), - /*modifiedKeys=*/ ImmutableSet.of(key), + /* tsgm= */ null, + inMemoryGraph, + /* modifiedKeys= */ ImmutableSet.of(key), fsvcThreads, - syscallCache); + syscallCache, + skyValueDirtinessChecker); assertThat(diff.changedKeysWithNewValues()).containsExactly(key, delta); assertThat(diff.changedKeysWithoutNewValues()) @@ -320,16 +281,17 @@ public void getDiffWithInferredAncestors_deleteFile_returnsFileParentListing() t FileStateKey key = fileStateValueKey("dir/file1"); FileStateValue oldValue = fileStateValue("dir/file1"); file.delete(); + addDoneNodesAndThenMarkChanged( + ImmutableMap.of(key, oldValue, fileStateValueKey("dir"), fileStateValue("dir"))); ImmutableDiff diff = FileSystemValueCheckerInferringAncestors.getDiffWithInferredAncestors( - /*tsgm=*/ null, - /*graphValues=*/ ImmutableMap.of( - key, oldValue, fileStateValueKey("dir"), fileStateValue("dir")), - /*graphDoneValues=*/ ImmutableMap.of(), - /*modifiedKeys=*/ ImmutableSet.of(key), + /* tsgm= */ null, + inMemoryGraph, + /* modifiedKeys= */ ImmutableSet.of(key), fsvcThreads, - syscallCache); + syscallCache, + skyValueDirtinessChecker); assertThat(diff.changedKeysWithNewValues()) .containsExactly(key, NONEXISTENT_FILE_STATE_NODE_DELTA); @@ -345,18 +307,21 @@ public void getDiffWithInferredAncestors_deleteFileFromDirWithListing_skipsDirSt FileStateKey key = fileStateValueKey("dir/file1"); FileStateValue oldValue = fileStateValue("dir/file1"); file1.delete(); + addDoneNodesAndThenMarkChanged( + ImmutableMap.of(key, oldValue, fileStateValueKey("dir"), fileStateValue("dir"))); + addDoneNodes( + ImmutableMap.of( + directoryListingStateValueKey("dir"), + directoryListingStateValue(file("file1"), file("file2")))); ImmutableDiff diff = FileSystemValueCheckerInferringAncestors.getDiffWithInferredAncestors( - /*tsgm=*/ null, - /*graphValues=*/ ImmutableMap.of( - key, oldValue, fileStateValueKey("dir"), fileStateValue("dir")), - /*graphDoneValues=*/ ImmutableMap.of( - directoryListingStateValueKey("dir"), - directoryListingStateValue(file("file1"), file("file2"))), - /*modifiedKeys=*/ ImmutableSet.of(key), + /* tsgm= */ null, + inMemoryGraph, + /* modifiedKeys= */ ImmutableSet.of(key), fsvcThreads, - syscallCache); + syscallCache, + skyValueDirtinessChecker); assertThat(diff.changedKeysWithNewValues()) .containsExactly(key, NONEXISTENT_FILE_STATE_NODE_DELTA); @@ -372,21 +337,23 @@ public void getDiffWithInferredAncestors_deleteLastFileFromDir_ignoresInvalidate FileStateKey key = fileStateValueKey("dir/file1"); FileStateValue oldValue = fileStateValue("dir/file1"); file1.delete(); + addDoneNodesAndThenMarkChanged( + ImmutableMap.of( + key, + oldValue, + fileStateValueKey("dir"), + fileStateValue("dir"), + directoryListingStateValueKey("dir"), + directoryListingStateValue(file("file1"), file("file2")))); ImmutableDiff diff = FileSystemValueCheckerInferringAncestors.getDiffWithInferredAncestors( - /*tsgm=*/ null, - /*graphValues=*/ ImmutableMap.of( - key, - oldValue, - fileStateValueKey("dir"), - fileStateValue("dir"), - directoryListingStateValueKey("dir"), - directoryListingStateValue(file("file1"), file("file2"))), - /*graphDoneValues=*/ ImmutableMap.of(), - /*modifiedKeys=*/ ImmutableSet.of(key), + /* tsgm= */ null, + inMemoryGraph, + /* modifiedKeys= */ ImmutableSet.of(key), fsvcThreads, - syscallCache); + syscallCache, + skyValueDirtinessChecker); assertThat(diff.changedKeysWithNewValues()) .containsExactly(key, NONEXISTENT_FILE_STATE_NODE_DELTA); @@ -404,17 +371,21 @@ public void getDiffWithInferredAncestors_modifyAllUnknownEntriesInDirWithListing .createSymbolicLink(PathFragment.create("file")); FileStateKey fileKey = fileStateValueKey("dir/file"); FileStateKey symlinkKey = fileStateValueKey("dir/symlink"); + addDoneNodesAndThenMarkChanged( + ImmutableMap.of(fileStateValueKey("dir"), fileStateValue("dir"))); + addDoneNodes( + ImmutableMap.of( + directoryListingStateValueKey("dir"), + directoryListingStateValue(file("file"), symlink("symlink")))); ImmutableDiff diff = FileSystemValueCheckerInferringAncestors.getDiffWithInferredAncestors( - /*tsgm=*/ null, - /*graphValues=*/ ImmutableMap.of(fileStateValueKey("dir"), fileStateValue("dir")), - /*graphDoneValues=*/ ImmutableMap.of( - directoryListingStateValueKey("dir"), - directoryListingStateValue(file("file"), symlink("symlink"))), - /*modifiedKeys=*/ ImmutableSet.of(fileKey, symlinkKey), + /* tsgm= */ null, + inMemoryGraph, + /* modifiedKeys= */ ImmutableSet.of(fileKey, symlinkKey), fsvcThreads, - syscallCache); + syscallCache, + skyValueDirtinessChecker); assertThat(diff.changedKeysWithNewValues()) .containsExactly( @@ -434,16 +405,18 @@ public void getDiffWithInferredAncestors_replaceUnknownEntriesInDirWithListing_s FileStateKey file1Key = fileStateValueKey("dir/file1"); FileStateKey file2Key = fileStateValueKey("dir/file2"); DirectoryListingStateValue.Key dirKey = directoryListingStateValueKey("dir"); + addDoneNodesAndThenMarkChanged( + ImmutableMap.of(fileStateValueKey("dir"), fileStateValue("dir"))); + addDoneNodes(ImmutableMap.of(dirKey, directoryListingStateValue(file("file1"), file("file2")))); ImmutableDiff diff = FileSystemValueCheckerInferringAncestors.getDiffWithInferredAncestors( - /*tsgm=*/ null, - /*graphValues=*/ ImmutableMap.of(fileStateValueKey("dir"), fileStateValue("dir")), - /*graphDoneValues=*/ ImmutableMap.of( - dirKey, directoryListingStateValue(file("file1"), file("file2"))), - /*modifiedKeys=*/ ImmutableSet.of(file1Key, file2Key), + /* tsgm= */ null, + inMemoryGraph, + /* modifiedKeys= */ ImmutableSet.of(file1Key, file2Key), fsvcThreads, - syscallCache); + syscallCache, + skyValueDirtinessChecker); assertIsSubsetOf( diff.changedKeysWithNewValues().entrySet(), @@ -481,23 +454,25 @@ public void getDiffWithInferredAncestors_deleteAllFilesFromDir_returnsFilesAndDi file1.delete(); file2.delete(); file3.delete(); + addDoneNodesAndThenMarkChanged( + ImmutableMap.of( + key1, + oldValue1, + key2, + oldValue2, + key3, + oldValue3, + fileStateValueKey("dir"), + fileStateValue("dir"))); ImmutableDiff diff = FileSystemValueCheckerInferringAncestors.getDiffWithInferredAncestors( - /*tsgm=*/ null, - /*graphValues=*/ ImmutableMap.of( - key1, - oldValue1, - key2, - oldValue2, - key3, - oldValue3, - fileStateValueKey("dir"), - fileStateValue("dir")), - /*graphDoneValues=*/ ImmutableMap.of(), - /*modifiedKeys=*/ ImmutableSet.of(key1, key2, key3), + /* tsgm= */ null, + inMemoryGraph, + /* modifiedKeys= */ ImmutableSet.of(key1, key2, key3), fsvcThreads, - syscallCache); + syscallCache, + skyValueDirtinessChecker); assertThat(diff.changedKeysWithNewValues()) .containsExactly( @@ -520,23 +495,25 @@ public void getDiffWithInferredAncestors_deleteFileWithDirs_returnsFileAndDirs() FileStateKey abcFileKey = fileStateValueKey("a/b/c/file"); FileStateValue abcFileValue = fileStateValue("a/b/c/file"); scratch.dir("a/b").deleteTree(); + addDoneNodesAndThenMarkChanged( + ImmutableMap.of( + fileStateValueKey("a"), + fileStateValue("a"), + abKey, + abValue, + abcKey, + abcValue, + abcFileKey, + abcFileValue)); ImmutableDiff diff = FileSystemValueCheckerInferringAncestors.getDiffWithInferredAncestors( - /*tsgm=*/ null, - /*graphValues=*/ ImmutableMap.of( - fileStateValueKey("a"), - fileStateValue("a"), - abKey, - abValue, - abcKey, - abcValue, - abcFileKey, - abcFileValue), - /*graphDoneValues=*/ ImmutableMap.of(), - /*modifiedKeys=*/ ImmutableSet.of(abcFileKey), + /* tsgm= */ null, + inMemoryGraph, + /* modifiedKeys= */ ImmutableSet.of(abcFileKey), fsvcThreads, - syscallCache); + syscallCache, + skyValueDirtinessChecker); assertThat(diff.changedKeysWithNewValues()) .containsExactly( @@ -562,23 +539,25 @@ public void getDiffWithInferredAncestors_deleteFileWithReportedDirs_returnsFileA FileStateKey abcFileKey = fileStateValueKey("a/b/c/file"); FileStateValue abcFileValue = fileStateValue("a/b/c/file"); scratch.dir("a/b").deleteTree(); + addDoneNodesAndThenMarkChanged( + ImmutableMap.of( + fileStateValueKey("a"), + fileStateValue("a"), + abKey, + abValue, + abcKey, + abcValue, + abcFileKey, + abcFileValue)); ImmutableDiff diff = FileSystemValueCheckerInferringAncestors.getDiffWithInferredAncestors( - /*tsgm=*/ null, - /*graphValues=*/ ImmutableMap.of( - fileStateValueKey("a"), - fileStateValue("a"), - abKey, - abValue, - abcKey, - abcValue, - abcFileKey, - abcFileValue), - /*graphDoneValues=*/ ImmutableMap.of(), - /*modifiedKeys=*/ ImmutableSet.of(abcFileKey, abKey), + /* tsgm= */ null, + inMemoryGraph, + /* modifiedKeys= */ ImmutableSet.of(abcFileKey, abKey), fsvcThreads, - syscallCache); + syscallCache, + skyValueDirtinessChecker); assertThat(diff.changedKeysWithNewValues()) .containsExactly( @@ -604,21 +583,23 @@ public void getDiffWithInferredAncestors_deleteFile_infersDirFromModifiedSibling FileStateValue file2Value = fileStateValue("dir/file2"); file1.delete(); scratch.overwriteFile("dir/file2", "12"); + addDoneNodesAndThenMarkChanged( + ImmutableMap.of( + fileStateValueKey("dir"), + fileStateValue("dir"), + file1Key, + file1Value, + file2Key, + file2Value)); ImmutableDiff diff = FileSystemValueCheckerInferringAncestors.getDiffWithInferredAncestors( - /*tsgm=*/ null, - /*graphValues=*/ ImmutableMap.of( - fileStateValueKey("dir"), - fileStateValue("dir"), - file1Key, - file1Value, - file2Key, - file2Value), - /*graphDoneValues=*/ ImmutableMap.of(), - /*modifiedKeys=*/ ImmutableSet.of(file1Key, file2Key, fileStateValueKey("dir")), + /* tsgm= */ null, + inMemoryGraph, + /* modifiedKeys= */ ImmutableSet.of(file1Key, file2Key, fileStateValueKey("dir")), fsvcThreads, - syscallCache); + syscallCache, + skyValueDirtinessChecker); Delta file2NewValue = fileStateValueDelta("dir/file2"); assertThat(diff.changedKeysWithNewValues()) @@ -639,23 +620,25 @@ public void getDiffWithInferredAncestors_deleteDirReportDirOnly_returnsDir() thr FileStateKey dirKey = fileStateValueKey("dir"); FileStateValue dirValue = fileStateValue("dir"); file1.getParentDirectory().deleteTree(); + addDoneNodesAndThenMarkChanged( + ImmutableMap.of( + file1Key, + file1Value, + file2Key, + file2Value, + dirKey, + dirValue, + fileStateValueKey(""), + fileStateValue(""))); ImmutableDiff diff = FileSystemValueCheckerInferringAncestors.getDiffWithInferredAncestors( - /*tsgm=*/ null, - /*graphValues=*/ ImmutableMap.of( - file1Key, - file1Value, - file2Key, - file2Value, - dirKey, - dirValue, - fileStateValueKey(""), - fileStateValue("")), - /*graphDoneValues=*/ ImmutableMap.of(), - /*modifiedKeys=*/ ImmutableSet.of(dirKey), + /* tsgm= */ null, + inMemoryGraph, + /* modifiedKeys= */ ImmutableSet.of(dirKey), fsvcThreads, - syscallCache); + syscallCache, + skyValueDirtinessChecker); assertThat(diff.changedKeysWithNewValues()) .containsExactly(dirKey, NONEXISTENT_FILE_STATE_NODE_DELTA); @@ -667,15 +650,17 @@ public void getDiffWithInferredAncestors_deleteDirReportDirOnly_returnsDir() thr @Test public void getDiffWithInferredAncestors_phantomChangeForNonexistentEntry_returnsEmptyDiff() throws Exception { + addDoneNodesAndThenMarkChanged( + ImmutableMap.of(fileStateValueKey("file"), NONEXISTENT_FILE_STATE_NODE)); + ImmutableDiff diff = FileSystemValueCheckerInferringAncestors.getDiffWithInferredAncestors( - /*tsgm=*/ null, - /*graphValues=*/ ImmutableMap.of( - fileStateValueKey("file"), NONEXISTENT_FILE_STATE_NODE), - /*graphDoneValues=*/ ImmutableMap.of(), - /*modifiedKeys=*/ ImmutableSet.of(fileStateValueKey("file")), + /* tsgm= */ null, + inMemoryGraph, + /* modifiedKeys= */ ImmutableSet.of(fileStateValueKey("file")), fsvcThreads, - syscallCache); + syscallCache, + skyValueDirtinessChecker); assertThat(diff.changedKeysWithoutNewValues()).isEmpty(); assertThat(diff.changedKeysWithNewValues()).isEmpty(); @@ -683,23 +668,22 @@ public void getDiffWithInferredAncestors_phantomChangeForNonexistentEntry_return } @Test - public void getDiffWithInferredAncestors_statFails_fails() { + public void getDiffWithInferredAncestors_statFails_fails() throws Exception { throwOnStat = new IOException("oh no"); - ImmutableMap graphValues = - ImmutableMap.of(fileStateValueKey("file"), NONEXISTENT_FILE_STATE_NODE); - ImmutableSet modifiedKeys = ImmutableSet.of(fileStateValueKey("file")); + addDoneNodesAndThenMarkChanged( + ImmutableMap.of(fileStateValueKey("file"), NONEXISTENT_FILE_STATE_NODE)); AbruptExitException e = assertThrows( AbruptExitException.class, () -> FileSystemValueCheckerInferringAncestors.getDiffWithInferredAncestors( - /*tsgm=*/ null, - graphValues, - /*graphDoneValues=*/ ImmutableMap.of(), - modifiedKeys, + /* tsgm= */ null, + inMemoryGraph, + /* modifiedKeys= */ ImmutableSet.of(fileStateValueKey("file")), fsvcThreads, - syscallCache)); + syscallCache, + skyValueDirtinessChecker)); assertThat(e.getDetailedExitCode().getFailureDetail().hasDiffAwareness()).isTrue(); assertThat(e.getDetailedExitCode().getFailureDetail().getDiffAwareness().getCode()) @@ -708,22 +692,21 @@ public void getDiffWithInferredAncestors_statFails_fails() { } @Test - public void getDiffWithInferredAncestors_statCrashes_fails() { + public void getDiffWithInferredAncestors_statCrashes_fails() throws Exception { throwOnStat = new RuntimeException("oh no"); - ImmutableMap graphValues = - ImmutableMap.of(fileStateValueKey("file"), NONEXISTENT_FILE_STATE_NODE); - ImmutableSet modifiedKeys = ImmutableSet.of(fileStateValueKey("file")); + addDoneNodesAndThenMarkChanged( + ImmutableMap.of(fileStateValueKey("file"), NONEXISTENT_FILE_STATE_NODE)); assertThrows( IllegalStateException.class, () -> FileSystemValueCheckerInferringAncestors.getDiffWithInferredAncestors( - /*tsgm=*/ null, - graphValues, - /*graphDoneValues=*/ ImmutableMap.of(), - modifiedKeys, + /* tsgm= */ null, + inMemoryGraph, + /* modifiedKeys= */ ImmutableSet.of(fileStateValueKey("file")), fsvcThreads, - syscallCache)); + syscallCache, + skyValueDirtinessChecker)); } private static void assertIsSubsetOf(Iterable list, T... elements) { @@ -733,29 +716,11 @@ private static void assertIsSubsetOf(Iterable list, T... elements) { .containsAtLeastElementsIn(list); } - private FileStateKey fileStateValueKey(String relativePath) { - return FileStateValue.key( - RootedPath.toRootedPath(root, root.asPath().getRelative(relativePath))); - } - - private DirectoryListingStateValue.Key directoryListingStateValueKey(String relativePath) { - return DirectoryListingStateValue.key( - RootedPath.toRootedPath(root, root.asPath().getRelative(relativePath))); - } - - private static DirectoryListingStateValue directoryListingStateValue(Dirent... dirents) { - return DirectoryListingStateValue.create(ImmutableList.copyOf(dirents)); - } - - private FileStateValue fileStateValue(String relativePath) throws IOException { - return FileStateValue.create( - RootedPath.toRootedPath( - untrackedRoot, untrackedRoot.asPath().asFragment().getRelative(relativePath)), - SyscallCache.NO_CACHE, - /*tsgm=*/ null); - } - private Delta fileStateValueDelta(String relativePath) throws IOException { return Delta.justNew(fileStateValue(relativePath)); } + + private void addDoneNodes(ImmutableMap values) throws InterruptedException { + addDoneNodes(values, /* mtsv= */ null); + } } diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestorsTestBase.java b/src/test/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestorsTestBase.java new file mode 100644 index 00000000000000..aa871af188344f --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/skyframe/FileSystemValueCheckerInferringAncestorsTestBase.java @@ -0,0 +1,144 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.fail; + +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.FileStateValue; +import com.google.devtools.build.lib.testutil.Scratch; +import com.google.devtools.build.lib.vfs.DelegateFileSystem; +import com.google.devtools.build.lib.vfs.Dirent; +import com.google.devtools.build.lib.vfs.FileStateKey; +import com.google.devtools.build.lib.vfs.FileStatus; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.Root; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.lib.vfs.SyscallCache; +import com.google.devtools.build.skyframe.InMemoryGraph; +import com.google.devtools.build.skyframe.InMemoryNodeEntry; +import com.google.devtools.build.skyframe.NodeBatch; +import com.google.devtools.build.skyframe.NodeEntry.DirtyType; +import com.google.devtools.build.skyframe.QueryableGraph.Reason; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import com.google.devtools.build.skyframe.Version; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map.Entry; +import javax.annotation.Nullable; +import org.junit.After; +import org.junit.Before; + +public class FileSystemValueCheckerInferringAncestorsTestBase { + protected final Scratch scratch = new Scratch(); + protected final List statedPaths = new ArrayList<>(); + protected DefaultSyscallCache syscallCache = DefaultSyscallCache.newBuilder().build(); + protected Root root; + protected final InMemoryGraph inMemoryGraph = InMemoryGraph.create(); + + private Root untrackedRoot; + Exception throwOnStat; + + @Before + public void createRoot() throws IOException { + Path srcRootPath = scratch.dir("/src"); + PathFragment srcRoot = srcRootPath.asFragment(); + FileSystem trackingFileSystem = + new DelegateFileSystem(scratch.getFileSystem()) { + @Nullable + @Override + public synchronized FileStatus statIfFound(PathFragment path, boolean followSymlinks) + throws IOException { + if (throwOnStat != null) { + Exception toThrow = throwOnStat; + throwOnStat = null; + Throwables.propagateIfPossible(toThrow, IOException.class); + fail("Unexpected exception type"); + } + statedPaths.add(path.relativeTo(srcRoot).toString()); + return super.statIfFound(path, followSymlinks); + } + }; + root = Root.fromPath(trackingFileSystem.getPath(srcRoot)); + scratch.setWorkingDir("/src"); + untrackedRoot = Root.fromPath(srcRootPath); + } + + @After + public void checkExceptionThrown() { + assertThat(throwOnStat).isNull(); + syscallCache.clear(); + } + + protected FileStateKey fileStateValueKey(String relativePath) { + return FileStateValue.key( + RootedPath.toRootedPath(root, root.asPath().getRelative(relativePath))); + } + + protected DirectoryListingStateValue.Key directoryListingStateValueKey(String relativePath) { + return DirectoryListingStateValue.key( + RootedPath.toRootedPath(root, root.asPath().getRelative(relativePath))); + } + + protected FileStateValue fileStateValue(String relativePath) throws IOException { + return FileStateValue.create( + RootedPath.toRootedPath( + untrackedRoot, untrackedRoot.asPath().asFragment().getRelative(relativePath)), + SyscallCache.NO_CACHE, + /* tsgm= */ null); + } + + protected static DirectoryListingStateValue directoryListingStateValue(Dirent... dirents) { + return DirectoryListingStateValue.create(ImmutableList.copyOf(dirents)); + } + + protected void addDoneNodesAndThenMarkChanged(ImmutableMap values) + throws InterruptedException { + addDoneNodesAndThenMarkChanged(values, /* mtsv= */ null); + } + + protected void addDoneNodesAndThenMarkChanged( + ImmutableMap values, @Nullable Version mtsv) throws InterruptedException { + for (Entry entry : values.entrySet()) { + InMemoryNodeEntry node = addDoneNode(entry.getKey(), entry.getValue(), mtsv); + node.markDirty(DirtyType.CHANGE); + } + } + + protected void addDoneNodes(ImmutableMap values, @Nullable Version mtsv) + throws InterruptedException { + for (Entry entry : values.entrySet()) { + addDoneNode(entry.getKey(), entry.getValue(), mtsv); + } + } + + @CanIgnoreReturnValue + private InMemoryNodeEntry addDoneNode(SkyKey key, SkyValue value, @Nullable Version mtsv) + throws InterruptedException { + NodeBatch batch = inMemoryGraph.createIfAbsentBatch(null, Reason.OTHER, ImmutableList.of(key)); + InMemoryNodeEntry entry = (InMemoryNodeEntry) batch.get(key); + entry.addReverseDepAndCheckIfDone(null); + entry.markRebuilding(); + entry.setValue(value, Version.minimal(), /* maxTransitiveSourceVersion= */ mtsv); + return entry; + } +} diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/LocalDiffAwarenessIntegrationTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/LocalDiffAwarenessIntegrationTest.java index e1ba1f8cd3c0e3..72e93e2485d1ce 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/LocalDiffAwarenessIntegrationTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/LocalDiffAwarenessIntegrationTest.java @@ -132,7 +132,7 @@ public void changedFile_statFails_throwsError() throws Exception { AbruptExitException.class, () -> buildTargetWithRetryUntilSeesChange("//foo", "foo/BUILD")); - assertThat(e).hasCauseThat().hasCauseThat().hasCauseThat().isSameInstanceAs(injectedException); + assertThat(e).hasCauseThat().hasCauseThat().hasCauseThat().isInstanceOf(IOException.class); } /** diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java index 0def6666f83855..1853972778e0dd 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java @@ -27,6 +27,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; import com.google.devtools.build.lib.bazel.bzlmod.Version; +import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsUtil; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; @@ -243,8 +244,7 @@ protected ImmutableList extraPrecomputedValues() { PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false), PrecomputedValue.injected( BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), - PrecomputedValue.injected( - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), + PrecomputedValue.injected(YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunctionTest.java index 06672b18ddc308..96fada31eed435 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunctionTest.java @@ -31,6 +31,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.Version; +import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsUtil; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; @@ -82,8 +83,7 @@ protected ImmutableList extraPrecomputedValues() thro ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl())), PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false), PrecomputedValue.injected(ModuleFileFunction.MODULE_OVERRIDES, ImmutableMap.of()), - PrecomputedValue.injected( - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), + PrecomputedValue.injected(YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), PrecomputedValue.injected( BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/TreeArtifactValueTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/TreeArtifactValueTest.java index d855ed3efa6c0c..e950edc6189563 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/TreeArtifactValueTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/TreeArtifactValueTest.java @@ -17,8 +17,8 @@ import static com.google.common.truth.Truth8.assertThat; import static org.junit.Assert.assertThrows; import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; @@ -734,9 +734,9 @@ private static FileArtifactValue metadataWithId(int id) { } private static FileArtifactValue metadataWithIdNoDigest(int id) { - FileArtifactValue value = mock(FileArtifactValue.class); - when(value.getDigest()).thenReturn(null); - when(value.getModifiedTime()).thenReturn((long) id); + FileArtifactValue value = spy(FileArtifactValue.class); + doReturn(null).when(value).getDigest(); + doReturn((long) id).when(value).getModifiedTime(); return value; } } diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/toolchains/RegisteredExecutionPlatformsFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/toolchains/RegisteredExecutionPlatformsFunctionTest.java index 162eb7cbbe6095..2a3ef2fbab270a 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/toolchains/RegisteredExecutionPlatformsFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/toolchains/RegisteredExecutionPlatformsFunctionTest.java @@ -29,6 +29,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; +import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsUtil; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; @@ -109,8 +110,7 @@ protected ImmutableList extraPrecomputedValues() { ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl())), PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false), PrecomputedValue.injected(ModuleFileFunction.MODULE_OVERRIDES, ImmutableMap.of()), - PrecomputedValue.injected( - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), + PrecomputedValue.injected(YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), PrecomputedValue.injected( BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/toolchains/RegisteredToolchainsFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/toolchains/RegisteredToolchainsFunctionTest.java index 49b331c039d1d6..9bb149377d1c81 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/toolchains/RegisteredToolchainsFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/toolchains/RegisteredToolchainsFunctionTest.java @@ -27,6 +27,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; +import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsUtil; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; @@ -64,8 +65,7 @@ protected ImmutableList extraPrecomputedValues() { ModuleFileFunction.REGISTRIES, ImmutableList.of(registry.getUrl())), PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, false), PrecomputedValue.injected(ModuleFileFunction.MODULE_OVERRIDES, ImmutableMap.of()), - PrecomputedValue.injected( - BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), + PrecomputedValue.injected(YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), PrecomputedValue.injected( BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( diff --git a/src/test/java/com/google/devtools/build/lib/starlark/StarlarkIntegrationTest.java b/src/test/java/com/google/devtools/build/lib/starlark/StarlarkIntegrationTest.java index b698efac1d597f..ee5f77ff6760be 100644 --- a/src/test/java/com/google/devtools/build/lib/starlark/StarlarkIntegrationTest.java +++ b/src/test/java/com/google/devtools/build/lib/starlark/StarlarkIntegrationTest.java @@ -2908,10 +2908,8 @@ public void testPrintFromTransitionImpl() throws Exception { scratch.file( "test/rules.bzl", "def _transition_impl(settings, attr):", - " foo = settings['//command_line_option:foo']", - " print('printing from transition impl', foo)", - " foo = foo if foo.endswith('meowmeowmeow') else foo + 'meow'", - " return {'//command_line_option:foo': foo}", + " print('printing from transition impl', settings['//command_line_option:foo'])", + " return {'//command_line_option:foo': " + "settings['//command_line_option:foo']+'meow'}", "my_transition = transition(", " implementation = _transition_impl,", " inputs = ['//command_line_option:foo'],", diff --git a/src/test/java/com/google/devtools/build/lib/starlark/util/BazelEvaluationTestCase.java b/src/test/java/com/google/devtools/build/lib/starlark/util/BazelEvaluationTestCase.java index c8d817893dea32..2afbb508886ff0 100644 --- a/src/test/java/com/google/devtools/build/lib/starlark/util/BazelEvaluationTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/starlark/util/BazelEvaluationTestCase.java @@ -44,7 +44,6 @@ import net.starlark.java.eval.Starlark; import net.starlark.java.eval.StarlarkSemantics; import net.starlark.java.eval.StarlarkThread; -import net.starlark.java.syntax.Expression; import net.starlark.java.syntax.FileOptions; import net.starlark.java.syntax.ParserInput; import net.starlark.java.syntax.SyntaxError; @@ -83,15 +82,6 @@ public ExtendedEventHandler getEventHandler() { return eventCollectionApparatus.reporter(); } - // TODO(adonovan): don't let subclasses inherit vaguely specified "helpers". - // Separate all the tests clearly into tests of the scanner, parser, resolver, - // and evaluation. - - /** Parses an expression. */ - final Expression parseExpression(String... lines) throws SyntaxError.Exception { - return Expression.parse(ParserInput.fromLines(lines)); - } - /** Updates a global binding in the module. */ // TODO(adonovan): rename setGlobal. @CanIgnoreReturnValue @@ -191,14 +181,6 @@ public void checkEvalErrorContains(String msg, String... input) throws Exception } } - public void checkEvalErrorDoesNotContain(String msg, String... input) throws Exception { - try { - exec(input); - } catch (SyntaxError.Exception | EvalException | EventCollectionApparatus.FailFastException e) { - assertThat(e).hasMessageThat().doesNotContain(msg); - } - } - // Forward relevant methods to the EventCollectionApparatus @CanIgnoreReturnValue public BazelEvaluationTestCase setFailFast(boolean failFast) { @@ -206,12 +188,6 @@ public BazelEvaluationTestCase setFailFast(boolean failFast) { return this; } - @CanIgnoreReturnValue - public BazelEvaluationTestCase assertNoWarningsOrErrors() { - eventCollectionApparatus.assertNoWarningsOrErrors(); - return this; - } - public EventCollector getEventCollector() { return eventCollectionApparatus.collector(); } @@ -220,20 +196,6 @@ public Event assertContainsError(String expectedMessage) { return eventCollectionApparatus.assertContainsError(expectedMessage); } - public Event assertContainsWarning(String expectedMessage) { - return eventCollectionApparatus.assertContainsWarning(expectedMessage); - } - - public Event assertContainsDebug(String expectedMessage) { - return eventCollectionApparatus.assertContainsDebug(expectedMessage); - } - - @CanIgnoreReturnValue - public BazelEvaluationTestCase clearEvents() { - eventCollectionApparatus.clear(); - return this; - } - /** Encapsulates a separate test which can be executed by a Scenario. */ protected interface Testable { void run() throws Exception; @@ -297,13 +259,6 @@ public Scenario testExpression(String src, Object expected) throws Exception { return this; } - /** Evaluates an expression and compares its result to the ordered list of expected objects. */ - @CanIgnoreReturnValue - public Scenario testExactOrder(String src, Object... items) throws Exception { - runTest(collectionTestable(src, items)); - return this; - } - /** Evaluates an expression and checks whether it fails with the expected error. */ @CanIgnoreReturnValue public Scenario testIfExactError(String expectedError, String... lines) throws Exception { @@ -318,13 +273,6 @@ public Scenario testIfErrorContains(String expectedError, String... lines) throw return this; } - /** Looks up the value of the specified variable and compares it to the expected value. */ - @CanIgnoreReturnValue - public Scenario testLookup(String name, Object expected) throws Exception { - runTest(createLookUpTestable(name, expected)); - return this; - } - /** * Creates a Testable that checks whether the evaluation of the given expression fails with the * expected error. @@ -345,19 +293,6 @@ public void run() throws Exception { }; } - /** - * Creates a Testable that checks whether the value of the expression is a sequence containing - * the expected elements. - */ - private Testable collectionTestable(final String src, final Object... expected) { - return new Testable() { - @Override - public void run() throws Exception { - assertThat((Iterable) eval(src)).containsExactly(expected).inOrder(); - } - }; - } - /** * Creates a testable that compares the value of the expression to a specified result. * @@ -386,23 +321,6 @@ public void run() throws Exception { }; } - /** - * Creates a Testable that looks up the given variable and compares its value to the expected - * value - * - * @param name - * @param expected - * @return An instance of Testable that does both lookup and comparison - */ - private Testable createLookUpTestable(final String name, final Object expected) { - return new Testable() { - @Override - public void run() throws Exception { - assertThat(lookup(name)).isEqualTo(expected); - } - }; - } - /** Executes the given Testable */ void runTest(Testable testable) throws Exception { run(new TestableDecorator(setup, testable)); diff --git a/src/test/java/com/google/devtools/build/lib/testutil/BUILD b/src/test/java/com/google/devtools/build/lib/testutil/BUILD index 324f4d7024c51d..19161d31b0b6f9 100644 --- a/src/test/java/com/google/devtools/build/lib/testutil/BUILD +++ b/src/test/java/com/google/devtools/build/lib/testutil/BUILD @@ -73,6 +73,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs", "//src/main/java/net/starlark/java/syntax", "//src/main/protobuf:failure_details_java_proto", + "//third_party:error_prone_annotations", "//third_party:guava", "//third_party:jsr305", "//third_party:junit4", diff --git a/src/test/java/com/google/devtools/build/lib/testutil/FakeAttributeMapper.java b/src/test/java/com/google/devtools/build/lib/testutil/FakeAttributeMapper.java index f050ed4069f83d..2e3c7750951929 100644 --- a/src/test/java/com/google/devtools/build/lib/testutil/FakeAttributeMapper.java +++ b/src/test/java/com/google/devtools/build/lib/testutil/FakeAttributeMapper.java @@ -21,6 +21,7 @@ import com.google.devtools.build.lib.packages.Attribute; import com.google.devtools.build.lib.packages.AttributeMap; import com.google.devtools.build.lib.packages.DependencyFilter; +import com.google.devtools.build.lib.packages.PackageArgs; import com.google.devtools.build.lib.packages.Type; import java.util.Map; import java.util.function.BiConsumer; @@ -110,23 +111,8 @@ public void visitLabels(String attributeName, Consumer