diff --git a/site/en/_book.yaml b/site/en/_book.yaml index 19bdc921b9d8aa..d4d20f4aaf9ea1 100644 --- a/site/en/_book.yaml +++ b/site/en/_book.yaml @@ -108,10 +108,6 @@ upper_tabs: path: /build/style-guide - title: Share variables path: /build/share-variables - - title: External dependencies - path: /build/external - - title: Manage dependencies with Bzlmod - path: /build/bzlmod - title: Recommended rules path: /community/recommended-rules - heading: Running Bazel @@ -136,6 +132,8 @@ upper_tabs: path: /external/extension - title: Module lockfile path: /external/lockfile + - title: '`mod` command' + path: /external/mod-command - title: Bzlmod migration guide path: /external/migration - title: Advanced topics diff --git a/site/en/about/roadmap.md b/site/en/about/roadmap.md index edb57cb66b2daa..99af4d1eb0148b 100644 --- a/site/en/about/roadmap.md +++ b/site/en/about/roadmap.md @@ -16,7 +16,7 @@ Q4 brings Bazel 6.0 — the new [long term support (LTS)](https://bazel.build/re ### Bzlmod: external dependency management system -[Bzlmod](https://bazel.build/docs/bzlmod) automatically resolves transitive dependencies, allowing projects to scale while staying fast and resource-efficient. Introduced experimentally in Bazel 5.0, Bzlmod will be generally available and provide a solution for the [diamond dependency problem](https://docs.google.com/document/d/1moQfNcEIttsk6vYanNKIy3ZuK53hQUFq1b1r0rmsYVg/edit#heading=h.lgyp7ubwxmjc). +[Bzlmod](https://bazel.build/external/overview#bzlmod) automatically resolves transitive dependencies, allowing projects to scale while staying fast and resource-efficient. Introduced experimentally in Bazel 5.0, Bzlmod will be generally available and provide a solution for the [diamond dependency problem](https://docs.google.com/document/d/1moQfNcEIttsk6vYanNKIy3ZuK53hQUFq1b1r0rmsYVg/edit#heading=h.lgyp7ubwxmjc). * Bzlmod goes from ‘experimental’ to ‘generally available’ * Includes support for `rules\_jvm\_external`, allowing users to download Maven dependencies for Java projects diff --git a/site/en/build/bzlmod.md b/site/en/build/bzlmod.md deleted file mode 100644 index bcf89dd3e36a38..00000000000000 --- a/site/en/build/bzlmod.md +++ /dev/null @@ -1,443 +0,0 @@ -Project: /_project.yaml -Book: /_book.yaml - -# Manage external dependencies with Bzlmod - -*Bzlmod* is the codename of the new [external dependency](/docs/external) system -introduced in Bazel 5.0. It was introduced to address several pain points of the -old system that couldn't feasibly be fixed incrementally; see the -[Problem Statement section of the original design doc](https://docs.google.com/document/d/1moQfNcEIttsk6vYanNKIy3ZuK53hQUFq1b1r0rmsYVg/edit#heading=h.xxnnwabymk1v){: .external} -for more details. - -In Bazel 5.0, Bzlmod is not turned on by default; the flag -`--experimental_enable_bzlmod` needs to be specified for the following to take -effect. As the flag name suggests, this feature is currently *experimental*; -APIs and behaviors may change until the feature officially launches. - -To migrate your project to Bzlmod, follow the [Bzlmod Migration Guide](https://docs.google.com/document/d/1JtXIVnXyFZ4bmbiBCr5gsTH4-opZAFf5DMMb-54kES0/edit?usp=sharing). -You can also find example Bzlmod usages in the [examples](https://github.com/bazelbuild/examples/tree/main/bzlmod) repository. - -## Bazel Modules {:#modules} - -The old `WORKSPACE`-based external dependency system is centered around -*repositories* (or *repos*), created via *repository rules* (or *repo rules*). -While repos are still an important concept in the new system, *modules* are the -core units of dependency. - -A *module* is essentially a Bazel project that can have multiple versions, each -of which publishes metadata about other modules that it depends on. This is -analogous to familiar concepts in other dependency management systems: a Maven -*artifact*, an npm *package*, a Cargo *crate*, a Go *module*, etc. - -A module simply specifies its dependencies using `name` and `version` pairs, -instead of specific URLs in `WORKSPACE`. The dependencies are then looked up in -a [Bazel registry](#registries); by default, the -[Bazel Central Registry](#bazel-central-registry). In your workspace, each -module then gets turned into a repo. - -### MODULE.bazel {:#module-bazel} - -Every version of every module has a `MODULE.bazel` file declaring its -dependencies and other metadata. Here's a basic example: - -```python -module( - name = "my-module", - version = "1.0", -) - -bazel_dep(name = "rules_cc", version = "0.0.1") -bazel_dep(name = "protobuf", version = "3.19.0") -``` - -The `MODULE.bazel` file should be located at the root of the workspace directory -(next to the `WORKSPACE` file). Unlike with the `WORKSPACE` file, you don't need -to specify your *transitive* dependencies; instead, you should only specify -*direct* dependencies, and the `MODULE.bazel` files of your dependencies are -processed to discover transitive dependencies automatically. - -The `MODULE.bazel` file is similar to `BUILD` files as it doesn't support any -form of control flow; it additionally forbids `load` statements. The directives -`MODULE.bazel` files support are: - -* [`module`](/rules/lib/globals#module), to specify metadata - about the current module, including its name, version, and so on; -* [`bazel_dep`](/rules/lib/globals#bazel_dep), to specify direct - dependencies on other Bazel modules; -* Overrides, which can only be used by the root module (that is, not by a - module which is being used as a dependency) to customize the behavior of a - certain direct or transitive dependency: - * [`single_version_override`](/rules/lib/globals#single_version_override) - * [`multiple_version_override`](/rules/lib/globals#multiple_version_override) - * [`archive_override`](/rules/lib/globals#archive_override) - * [`git_override`](/rules/lib/globals#git_override) - * [`local_path_override`](/rules/lib/globals#local_path_override) -* Directives related to [module extensions](#module-extensions): - * [`use_extension`](/rules/lib/globals#use_extension) - * [`use_repo`](/rules/lib/globals#use_repo) - -### Version format {:#version-format} - -Bazel has a diverse ecosystem and projects use various versioning schemes. The -most popular by far is [SemVer](https://semver.org){: .external}, but there are -also prominent projects using different schemes such as -[Abseil](https://github.com/abseil/abseil-cpp/releases){: .external}, whose -versions are date-based, for example `20210324.2`). - -For this reason, Bzlmod adopts a more relaxed version of the SemVer spec. The -differences include: - -* SemVer prescribes that the "release" part of the version must consist of 3 - segments: `MAJOR.MINOR.PATCH`. In Bazel, this requirement is loosened so - that any number of segments is allowed. -* In SemVer, each of the segments in the "release" part must be digits only. - In Bazel, this is loosened to allow letters too, and the comparison - semantics match the "identifiers" in the "prerelease" part. -* Additionally, the semantics of major, minor, and patch version increases are - not enforced. (However, see [compatibility level](#compatibility-level) for - details on how we denote backwards compatibility.) - -Any valid SemVer version is a valid Bazel module version. Additionally, two -SemVer versions `a` and `b` compare `a < b` iff the same holds when they're -compared as Bazel module versions. - -### Version resolution {:#version-resolution} - -The diamond dependency problem is a staple in the versioned dependency -management space. Suppose you have the following dependency graph: - -``` - A 1.0 - / \ - B 1.0 C 1.1 - | | - D 1.0 D 1.1 -``` - -Which version of D should be used? To resolve this question, Bzlmod uses the -[Minimal Version Selection](https://research.swtch.com/vgo-mvs){: .external} -(MVS) algorithm introduced in the Go module system. MVS assumes that all new -versions of a module are backwards compatible, and thus simply picks the highest -version specified by any dependent (D 1.1 in our example). It's called "minimal" -because D 1.1 here is the *minimal* version that could satisfy our requirements; -even if D 1.2 or newer exists, we don't select them. This has the added benefit -that the version selection is *high-fidelity* and *reproducible*. - -Version resolution is performed locally on your machine, not by the registry. - -### Compatibility level {:#compatibility-level} - -Note that MVS's assumption about backwards compatibility is feasible because it -simply treats backwards incompatible versions of a module as a separate module. -In terms of SemVer, that means A 1.x and A 2.x are considered distinct modules, -and can coexist in the resolved dependency graph. This is, in turn, made -possible by the fact that the major version is encoded in the package path in -Go, so there aren't any compile-time or linking-time conflicts. - -In Bazel, we don't have such guarantees. Thus we need a way to denote the "major -version" number in order to detect backwards incompatible versions. This number -is called the *compatibility level*, and is specified by each module version in -its `module()` directive. With this information in hand, we can throw an error -when we detect that versions of the same module with different compatibility -levels exist in the resolved dependency graph. - -### Repository names {:#repository-names} - -In Bazel, every external dependency has a repository name. Sometimes, the same -dependency might be used via different repository names (for example, both -`@io_bazel_skylib` and `@bazel_skylib` mean -[Bazel skylib](https://github.com/bazelbuild/bazel-skylib){: .external}), or the same -repository name might be used for different dependencies in different projects. - -In Bzlmod, repositories can be generated by Bazel modules and -[module extensions](#module-extensions). To resolve repository name conflicts, -we are embracing the [repository mapping](/docs/external#shadowing-dependencies) -mechanism in the new system. Here are two important concepts: - -* **Canonical repository name**: The globally unique repository name for each - repository. This will be the directory name the repository lives in. -
It's constructed as follows (**Warning**: the canonical name format is - not an API you should depend on, it's subject to change at any time): - - * For Bazel module repos: `{{ "" }}module_name{{ "" }}~{{ "" }}version{{ "" }}` -
(Example. `@bazel_skylib~1.0.3`) - * For module extension repos: `{{ "" }}module_name{{ "" }}~{{ "" }}version{{ "" }}~{{ "" }}extension_name{{ "" }}~{{ "" }}repo_name{{ "" }}` -
(Example. `@rules_cc~0.0.1~cc_configure~local_config_cc`) - -* **Apparent repository name**: The repository name to be used in the `BUILD` and - `.bzl` files within a repo. The same dependency could have different apparent - names in different repos. -
It's determined as follows: - - * For Bazel module repos: `{{ "" }}module_name{{ "" }}` by - default, or the name specified by the `repo_name` attribute in - [`bazel_dep`](/rules/lib/globals#bazel_dep). - * For module extension repos: repository name introduced via - [`use_repo`](/rules/lib/globals#use_repo). - -Every repository has a repository mapping dictionary of its direct dependencies, -which is a map from the apparent repository name to the canonical repository name. -We use the repository mapping to resolve the repository name when constructing a -label. Note that, there is no conflict of canonical repository names, and the -usages of apparent repository names can be discovered by parsing the `MODULE.bazel` -file, therefore conflicts can be easily caught and resolved without affecting -other dependencies. - -### Strict deps {:#strict-deps} - -The new dependency specification format allows us to perform stricter checks. In -particular, we now enforce that a module can only use repos created from its -direct dependencies. This helps prevent accidental and hard-to-debug breakages -when something in the transitive dependency graph changes. - -Strict deps is implemented based on -[repository mapping](/docs/external#shadowing-dependencies). Basically, the -repository mapping for each repo contains all of its *direct dependencies*, any -other repository is not visible. Visible dependencies for each repository are -determined as follows: - -* A Bazel module repo can see all repos introduced in the `MODULE.bazel` file - via [`bazel_dep`](/rules/lib/globals#bazel_dep) and - [`use_repo`](/rules/lib/globals#use_repo). -* A module extension repo can see all visible dependencies of the module that - provides the extension, plus all other repos generated by the same module - extension. - -## Registries {:#registries} - -Bzlmod discovers dependencies by requesting their information from Bazel -*registries*. A Bazel registry is simply a database of Bazel modules. The only -supported form of registries is an [*index registry*](#index-registry), which is -a local directory or a static HTTP server following a specific format. In the -future, we plan to add support for *single-module registries*, which are simply -git repos containing the source and history of a project. - -### Index registry {:#index-registry} - -An index registry is a local directory or a static HTTP server containing -information about a list of modules, including their homepage, maintainers, the -`MODULE.bazel` file of each version, and how to fetch the source of each -version. Notably, it does *not* need to serve the source archives itself. - -An index registry must follow the format below: - -* `/bazel_registry.json`: A JSON file containing metadata for the registry like: - * `mirrors`, specifying the list of mirrors to use for source archives. - * `module_base_path`, specifying the base path for modules with - `local_repository` type in the `source.json` file. -* `/modules`: A directory containing a subdirectory for each module in this - registry. -* `/modules/$MODULE`: A directory containing a subdirectory for each version - of this module, as well as the following file: - * `metadata.json`: A JSON file containing information about the module, - with the following fields: - * `homepage`: The URL of the project's homepage. - * `maintainers`: A list of JSON objects, each of which corresponds to - the information of a maintainer of the module *in the registry*. - Note that this is not necessarily the same as the *authors* of the - project. - * `versions`: A list of all the versions of this module to be found in - this registry. - * `yanked_versions`: A list of *yanked* versions of this module. This - is currently a no-op, but in the future, yanked versions will be - skipped or yield an error. -* `/modules/$MODULE/$VERSION`: A directory containing the following files: - * `MODULE.bazel`: The `MODULE.bazel` file of this module version. - * `source.json`: A JSON file containing information on how to fetch the - source of this module version. - * The default type is "archive" with the following fields: - * `url`: The URL of the source archive. - * `integrity`: The - [Subresource Integrity](https://w3c.github.io/webappsec-subresource-integrity/#integrity-metadata-description){: .external} - checksum of the archive. - * `strip_prefix`: A directory prefix to strip when extracting the - source archive. - * `patches`: A list of strings, each of which names a patch file to - apply to the extracted archive. The patch files are located under - the `/modules/$MODULE/$VERSION/patches` directory. - * `patch_strip`: Same as the `--strip` argument of Unix patch. - * The type can be changed to use a local path with these fields: - * `type`: `local_path` - * `path`: The local path to the repo, calculated as following: - * If path is an absolute path, will be used as it is. - * If path is a relative path and `module_base_path` is an absolute path, - path is resolved to `/` - * If path and `module_base_path` are both relative paths, path is - resolved to `//`. - Registry must be hosted locally and used by `--registry=file://`. - Otherwise, Bazel will throw an error. - * `patches/`: An optional directory containing patch files, only used when `source.json` has "archive" type. - -### Bazel Central Registry {:#bazel-central-registry} - -Bazel Central Registry (BCR) is an index registry located at -[bcr.bazel.build](https://bcr.bazel.build){: .external}. Its contents -are backed by the GitHub repo -[`bazelbuild/bazel-central-registry`](https://github.com/bazelbuild/bazel-central-registry){: .external}. - -The BCR is maintained by the Bazel community; contributors are welcome to submit -pull requests. See -[Bazel Central Registry Policies and Procedures](https://docs.google.com/document/d/1ReuBBp4EHnsuvcpfXM6ITDmP2lrOu8DGlePMUKvDnXM/edit?usp=sharing){: .external}. - -In addition to following the format of a normal index registry, the BCR requires -a `presubmit.yml` file for each module version -(`/modules/$MODULE/$VERSION/presubmit.yml`). This file specifies a few essential -build and test targets that can be used to sanity-check the validity of this -module version, and is used by the BCR's CI pipelines to ensure interoperability -between modules in the BCR. - -### Selecting registries {:#selecting-registries} - -The repeatable Bazel flag `--registry` can be used to specify the list of -registries to request modules from, so you could set up your project to fetch -dependencies from a third-party or internal registry. Earlier registries take -precedence. For convenience, you can put a list of `--registry` flags in the -`.bazelrc` file of your project. - -Note: If your registry is hosted on GitHub (for example, as a fork of -`bazelbuild/bazel-central-registry`) then your `--registry` value needs a raw -GitHub address under `raw.githubusercontent.com`. For example, on the `main` -branch of the `my-org` fork, you would set -`--registry=https://raw.githubusercontent.com/my-org/bazel-central-registry/main/`. - -## Module Extensions {:#module-extensions} - -Module extensions allow you to extend the module system by reading input data -from modules across the dependency graph, performing necessary logic to resolve -dependencies, and finally creating repos by calling repo rules. They are similar -in function to today's `WORKSPACE` macros, but are more suited in the world of -modules and transitive dependencies. - -Module extensions are defined in `.bzl` files, just like repo rules or -`WORKSPACE` macros. They're not invoked directly; rather, each module can -specify pieces of data called *tags* for extensions to read. Then, after module -version resolution is done, module extensions are run. Each extension is run -once after module resolution (still before any build actually happens), and -gets to read all the tags belonging to it across the entire dependency graph. - -``` - [ A 1.1 ] - [ * maven.dep(X 2.1) ] - [ * maven.pom(...) ] - / \ - bazel_dep / \ bazel_dep - / \ -[ B 1.2 ] [ C 1.0 ] -[ * maven.dep(X 1.2) ] [ * maven.dep(X 2.1) ] -[ * maven.dep(Y 1.3) ] [ * cargo.dep(P 1.1) ] - \ / - bazel_dep \ / bazel_dep - \ / - [ D 1.4 ] - [ * maven.dep(Z 1.4) ] - [ * cargo.dep(Q 1.1) ] -``` - -In the example dependency graph above, `A 1.1` and `B 1.2` etc are Bazel modules; -you can think of each one as a `MODULE.bazel` file. Each module can specify some -tags for module extensions; here some are specified for the extension "maven", -and some are specified for "cargo". When this dependency graph is finalized (for -example, maybe `B 1.2` actually has a `bazel_dep` on `D 1.3` but got upgraded to -`D 1.4` due to `C`), the extensions "maven" is run, and it gets to read all the -`maven.*` tags, using information therein to decide which repos to create. -Similarly for the "cargo" extension. - -### Extension usage {:#extension-usage} - -Extensions are hosted in Bazel modules themselves, so to use an extension in -your module, you need to first add a `bazel_dep` on that module, and then call -the [`use_extension`](/rules/lib/globals#use_extension) built-in -function to bring it into scope. Consider the following example, a snippet from -a `MODULE.bazel` file to use a hypothetical "maven" extension defined in the -`rules_jvm_external` module: - -```python -bazel_dep(name = "rules_jvm_external", version = "1.0") -maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven") -``` - -After bringing the extension into scope, you can then use the dot-syntax to -specify tags for it. Note that the tags need to follow the schema defined by the -corresponding *tag classes* (see [extension definition](#extension-definition) -below). Here's an example specifying some `maven.dep` and `maven.pom` tags. - -```python -maven.dep(coord="org.junit:junit:3.0") -maven.dep(coord="com.google.guava:guava:1.2") -maven.pom(pom_xml="//:pom.xml") -``` - -If the extension generates repos that you want to use in your module, use the -[`use_repo`](/rules/lib/globals#use_repo) directive to declare -them. This is to satisfy the strict deps condition and avoid local repo name -conflict. - -```python -use_repo( - maven, - "org_junit_junit", - guava="com_google_guava_guava", -) -``` - -The repos generated by an extension are part of its API, so from the tags you -specified, you should know that the "maven" extension is going to generate a -repo called "org_junit_junit", and one called "com_google_guava_guava". With -`use_repo`, you can optionally rename them in the scope of your module, like to -"guava" here. - -### Extension definition {:#extension-definition} - -Module extensions are defined similarly to repo rules, using the -[`module_extension`](/rules/lib/globals#module_extension) function. -Both have an implementation function; but while repo rules have a number of -attributes, module extensions have a number of -[`tag_class`es](/rules/lib/globals#tag_class), each of which has a -number of attributes. The tag classes define schemas for tags used by this -extension. Continuing our example of the hypothetical "maven" extension above: - -```python -# @rules_jvm_external//:extensions.bzl -maven_dep = tag_class(attrs = {"coord": attr.string()}) -maven_pom = tag_class(attrs = {"pom_xml": attr.label()}) -maven = module_extension( - implementation=_maven_impl, - tag_classes={"dep": maven_dep, "pom": maven_pom}, -) -``` - -These declarations make it clear that `maven.dep` and `maven.pom` tags can be -specified, using the attribute schema defined above. - -The implementation function is similar to a `WORKSPACE` macro, except that it -gets a [`module_ctx`](/rules/lib/module_ctx) object, which grants -access to the dependency graph and all pertinent tags. The implementation -function should then call repo rules to generate repos: - -```python -# @rules_jvm_external//:extensions.bzl -load("//:repo_rules.bzl", "maven_single_jar") -def _maven_impl(ctx): - coords = [] - for mod in ctx.modules: - coords += [dep.coord for dep in mod.tags.dep] - output = ctx.execute(["coursier", "resolve", coords]) # hypothetical call - repo_attrs = process_coursier(output) - [maven_single_jar(**attrs) for attrs in repo_attrs] -``` - -In the example above, we go through all the modules in the dependency graph -(`ctx.modules`), each of which is a -[`bazel_module`](/rules/lib/bazel_module) object whose `tags` field -exposes all the `maven.*` tags on the module. Then we invoke the CLI utility -Coursier to contact Maven and perform resolution. Finally, we use the resolution -result to create a number of repos, using the hypothetical `maven_single_jar` -repo rule. - -## External links - -* [Bazel External Dependencies Overhaul](https://docs.google.com/document/d/1moQfNcEIttsk6vYanNKIy3ZuK53hQUFq1b1r0rmsYVg/edit){: .external} - (original Bzlmod design doc) -* [Bazel Central Registry Policies and Procedures](https://docs.google.com/document/d/1ReuBBp4EHnsuvcpfXM6ITDmP2lrOu8DGlePMUKvDnXM/edit?usp=sharing){: .external} -* [Bazel Central Registry GitHub repo](https://github.com/bazelbuild/bazel-central-registry){: .external} -* [BazelCon 2021 talk on Bzlmod](https://www.youtube.com/watch?v=TxOCKtU39Fs){: .external} diff --git a/site/en/build/external.md b/site/en/build/external.md deleted file mode 100644 index 74d4b78260c16d..00000000000000 --- a/site/en/build/external.md +++ /dev/null @@ -1,368 +0,0 @@ -Project: /_project.yaml -Book: /_book.yaml - -# Working with External Dependencies - -Bazel can depend on targets from other projects. Dependencies from these other -projects are called _external dependencies_. - -Note: Bazel 5.0 and newer has a new external dependency system, codenamed -"Bzlmod", which renders a lot of the content on this page obsolete. See [Bzlmod -user guide](/docs/bzlmod) for more information. - -The `WORKSPACE` file (or `WORKSPACE.bazel` file) in the -[workspace directory](/concepts/build-ref#workspace) -tells Bazel how to get other projects' sources. These other projects can -contain one or more `BUILD` files with their own targets. `BUILD` files within -the main project can depend on these external targets by using their name from -the `WORKSPACE` file. - -For example, suppose there are two projects on a system: - -``` -/ - home/ - user/ - project1/ - WORKSPACE - BUILD - srcs/ - ... - project2/ - WORKSPACE - BUILD - my-libs/ -``` - -If `project1` wanted to depend on a target, `:foo`, defined in -`/home/user/project2/BUILD`, it could specify that a repository named -`project2` could be found at `/home/user/project2`. Then targets in -`/home/user/project1/BUILD` could depend on `@project2//:foo`. - -The `WORKSPACE` file allows users to depend on targets from other parts of the -filesystem or downloaded from the internet. It uses the same syntax as `BUILD` -files, but allows a different set of rules called _repository rules_ (sometimes -also known as _workspace rules_). Bazel comes with a few [built-in repository -rules](/reference/be/workspace) and a set of [embedded Starlark repository -rules](/rules/lib/repo/index). Users can also write [custom repository -rules](/rules/repository_rules) to get more complex behavior. - -## Supported types of external dependencies {:#types} - -A few basic types of external dependencies can be used: - -- [Dependencies on other Bazel projects](#bazel-projects) -- [Dependencies on non-Bazel projects](#non-bazel-projects) -- [Dependencies on external packages](#external-packages) - -### Depending on other Bazel projects {:#bazel-projects} - -If you want to use targets from a second Bazel project, you can -use -[`local_repository`](/reference/be/workspace#local_repository), -[`git_repository`](/rules/lib/repo/git#git_repository) -or [`http_archive`](/rules/lib/repo/http#http_archive) -to symlink it from the local filesystem, reference a git repository or download -it (respectively). - -For example, suppose you are working on a project, `my-project/`, and you want -to depend on targets from your coworker's project, `coworkers-project/`. Both -projects use Bazel, so you can add your coworker's project as an external -dependency and then use any targets your coworker has defined from your own -BUILD files. You would add the following to `my_project/WORKSPACE`: - -```python -local_repository( - name = "coworkers_project", - path = "/path/to/coworkers-project", -) -``` - -If your coworker has a target `//foo:bar`, your project can refer to it as -`@coworkers_project//foo:bar`. External project names must be -[valid workspace names](/rules/lib/globals#workspace). - -### Depending on non-Bazel projects {:#non-bazel-projects} - -Rules prefixed with `new_`, such as -[`new_local_repository`](/reference/be/workspace#new_local_repository), -allow you to create targets from projects that do not use Bazel. - -For example, suppose you are working on a project, `my-project/`, and you want -to depend on your coworker's project, `coworkers-project/`. Your coworker's -project uses `make` to build, but you'd like to depend on one of the .so files -it generates. To do so, add the following to `my_project/WORKSPACE`: - -```python -new_local_repository( - name = "coworkers_project", - path = "/path/to/coworkers-project", - build_file = "coworker.BUILD", -) -``` - -`build_file` specifies a `BUILD` file to overlay on the existing project, for -example: - -```python -cc_library( - name = "some-lib", - srcs = glob(["**"]), - visibility = ["//visibility:public"], -) -``` - -You can then depend on `@coworkers_project//:some-lib` from your project's -`BUILD` files. - -### Depending on external packages {:#external-packages} - -#### Maven artifacts and repositories {:#maven-repositories} - -Use the ruleset [`rules_jvm_external`](https://github.com/bazelbuild/rules_jvm_external){: .external} -to download artifacts from Maven repositories and make them available as Java -dependencies. - -## Fetching dependencies {:#fetching-dependencies} - -By default, external dependencies are fetched as needed during `bazel build`. If -you would like to prefetch the dependencies needed for a specific set of targets, use -[`bazel fetch`](/reference/command-line-reference#commands). -To unconditionally fetch all external dependencies, use -[`bazel sync`](/reference/command-line-reference#commands). -As fetched repositories are [stored in the output base](#layout), fetching -happens per workspace. - -## Shadowing dependencies {:#shadowing-dependencies} - -Whenever possible, it is recommended to have a single version policy in your -project. This is required for dependencies that you compile against and end up -in your final binary. But for cases where this isn't true, it is possible to -shadow dependencies. Consider the following scenario: - -myproject/WORKSPACE - -```python -workspace(name = "myproject") - -local_repository( - name = "A", - path = "../A", -) -local_repository( - name = "B", - path = "../B", -) -``` - -A/WORKSPACE - -```python -workspace(name = "A") - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -http_archive( - name = "testrunner", - urls = ["https://github.com/testrunner/v1.zip"], - sha256 = "...", -) -``` - -B/WORKSPACE - -```python -workspace(name = "B") - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -http_archive( - name = "testrunner", - urls = ["https://github.com/testrunner/v2.zip"], - sha256 = "..." -) -``` - -Both dependencies `A` and `B` depend on `testrunner`, but they depend on -different versions of `testrunner`. There is no reason for these test runners to -not peacefully coexist within `myproject`, however they will clash with each -other since they have the same name. To declare both dependencies, -update myproject/WORKSPACE: - -```python -workspace(name = "myproject") - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -http_archive( - name = "testrunner-v1", - urls = ["https://github.com/testrunner/v1.zip"], - sha256 = "..." -) -http_archive( - name = "testrunner-v2", - urls = ["https://github.com/testrunner/v2.zip"], - sha256 = "..." -) -local_repository( - name = "A", - path = "../A", - repo_mapping = {"@testrunner" : "@testrunner-v1"} -) -local_repository( - name = "B", - path = "../B", - repo_mapping = {"@testrunner" : "@testrunner-v2"} -) -``` - -This mechanism can also be used to join diamonds. For example if `A` and `B` -had the same dependency but call it by different names, those dependencies can -be joined in myproject/WORKSPACE. - -## Overriding repositories from the command line {:#overriding-repositories} - -To override a declared repository with a local repository from the command line, -use the -[`--override_repository`](/reference/command-line-reference#flag--override_repository) -flag. Using this flag changes the contents of external repositories without -changing your source code. - -For example, to override `@foo` to the local directory `/path/to/local/foo`, -pass the `--override_repository=foo=/path/to/local/foo` flag. - -Some of the use cases include: - -* Debugging issues. For example, you can override a `http_archive` repository - to a local directory where you can make changes more easily. -* Vendoring. If you are in an environment where you cannot make network calls, - override the network-based repository rules to point to local directories - instead. - -## Using proxies {:#using-proxies} - -Bazel will pick up proxy addresses from the `HTTPS_PROXY` and `HTTP_PROXY` -environment variables and use these to download HTTP/HTTPS files (if specified). - -## Support for IPv6 {:#support-for-ipv6} - -On IPv6-only machines, Bazel will be able to download dependencies with -no changes. On dual-stack IPv4/IPv6 machines, however, Bazel follows the same -convention as Java: if IPv4 is enabled, IPv4 is preferred. In some situations, -for example when IPv4 network is unable to resolve/reach external addresses, -this can cause `Network unreachable` exceptions and build failures. -In these cases, you can override Bazel's behavior to prefer IPv6 -by using [`java.net.preferIPv6Addresses=true` system property](https://docs.oracle.com/javase/8/docs/api/java/net/doc-files/net-properties.html){: .external}. -Specifically: - -* Use `--host_jvm_args=-Djava.net.preferIPv6Addresses=true` - [startup option](/docs/user-manual#startup-options), - for example by adding the following line in your - [`.bazelrc` file](/docs/bazelrc): - - `startup --host_jvm_args=-Djava.net.preferIPv6Addresses=true` - -* If you are running Java build targets that need to connect to the internet - as well (integration tests sometimes needs that), also use - `--jvmopt=-Djava.net.preferIPv6Addresses=true` - [tool flag](/docs/user-manual#jvmopt), for example by having the - following line in your [`.bazelrc` file](/docs/bazelrc): - - `build --jvmopt=-Djava.net.preferIPv6Addresses` - -* If you are using - [rules_jvm_external](https://github.com/bazelbuild/rules_jvm_external){: .external}, - for example, for dependency version resolution, also add - `-Djava.net.preferIPv6Addresses=true` to the `COURSIER_OPTS` - environment variable to [provide JVM options for Coursier](https://github.com/bazelbuild/rules_jvm_external#provide-jvm-options-for-coursier-with-coursier_opts){: .external} - -## Transitive dependencies {:#transitive-dependencies} - -Bazel only reads dependencies listed in your `WORKSPACE` file. If your project -(`A`) depends on another project (`B`) which lists a dependency on a third -project (`C`) in its `WORKSPACE` file, you'll have to add both `B` -and `C` to your project's `WORKSPACE` file. This requirement can balloon the -`WORKSPACE` file size, but limits the chances of having one library -include `C` at version 1.0 and another include `C` at 2.0. - -## Caching of external dependencies {:#caching-external-dependencies} - -By default, Bazel will only re-download external dependencies if their -definition changes. Changes to files referenced in the definition (such as patches -or `BUILD` files) are also taken into account by bazel. - -To force a re-download, use `bazel sync`. - -## Layout {:#layout} - -External dependencies are all downloaded to a directory under the subdirectory -`external` in the [output base](/remote/output-directories). In case of a -[local repository](/reference/be/workspace#local_repository), a symlink is created -there instead of creating a new directory. -You can see the `external` directory by running: - -```posix-terminal -ls $(bazel info output_base)/external -``` - -Note that running `bazel clean` will not actually delete the external -directory. To remove all external artifacts, use `bazel clean --expunge`. - -## Offline builds {:#offline-builds} - -It is sometimes desirable or necessary to run a build in an offline fashion. For -simple use cases, such as traveling on an airplane, -[prefetching](#fetching-dependencies) the needed -repositories with `bazel fetch` or `bazel sync` can be enough; moreover, the -using the option `--nofetch`, fetching of further repositories can be disabled -during the build. - -For true offline builds, where the providing of the needed files is to be done -by an entity different from bazel, bazel supports the option -`--distdir`. Whenever a repository rule asks bazel to fetch a file via -[`ctx.download`](/rules/lib/repository_ctx#download) or -[`ctx.download_and_extract`](/rules/lib/repository_ctx#download_and_extract) -and provides a hash sum of the file -needed, bazel will first look into the directories specified by that option for -a file matching the basename of the first URL provided, and use that local copy -if the hash matches. - -Bazel itself uses this technique to bootstrap offline from the [distribution -artifact](https://github.com/bazelbuild/bazel-website/blob/master/designs/_posts/2016-10-11-distribution-artifact.md). -It does so by [collecting all the needed external -dependencies](https://github.com/bazelbuild/bazel/blob/5cfa0303d6ac3b5bd031ff60272ce80a704af8c2/WORKSPACE#L116){: .external} -in an internal -[`distdir_tar`](https://github.com/bazelbuild/bazel/blob/5cfa0303d6ac3b5bd031ff60272ce80a704af8c2/distdir.bzl#L44){: .external}. - -However, bazel allows the execution of arbitrary commands in repository rules, -without knowing if they call out to the network. Therefore, bazel has no option -to enforce builds being fully offline. So testing if a build works correctly -offline requires external blocking of the network, as bazel does in its -bootstrap test. - -## Best practices {:#best-practices} - -### Repository rules {:#repository-rules} - -A repository rule should generally be responsible for: - -- Detecting system settings and writing them to files. -- Finding resources elsewhere on the system. -- Downloading resources from URLs. -- Generating or symlinking BUILD files into the external repository directory. - -Avoid using `repository_ctx.execute` when possible. For example, when using a non-Bazel C++ -library that has a build using Make, it is preferable to use `repository_ctx.download()` and then -write a BUILD file that builds it, instead of running `ctx.execute(["make"])`. - -Prefer [`http_archive`](/rules/lib/repo/http#http_archive) to `git_repository` and -`new_git_repository`. The reasons are: - -* Git repository rules depend on system `git(1)` whereas the HTTP downloader is built - into Bazel and has no system dependencies. -* `http_archive` supports a list of `urls` as mirrors, and `git_repository` supports only - a single `remote`. -* `http_archive` works with the [repository cache](/docs/build#repository-cache), but not - `git_repository`. See - [#5116](https://github.com/bazelbuild/bazel/issues/5116){: .external} for more information. - -Do not use `bind()`. See "[Consider removing -bind](https://github.com/bazelbuild/bazel/issues/1952){: .external}" for a long -discussion of its issues and alternatives. diff --git a/site/en/concepts/build-ref.md b/site/en/concepts/build-ref.md index 086a8c29509126..55a921a8a72ac5 100644 --- a/site/en/concepts/build-ref.md +++ b/site/en/concepts/build-ref.md @@ -3,59 +3,60 @@ Book: /_book.yaml # Workspaces, packages, and targets -Bazel builds software from source code organized in a directory tree called -a workspace. Source files in the workspace are organized in a nested -hierarchy of packages, where each package is a directory that contains a set -of related source files and one `BUILD` file. The `BUILD` file specifies what -software outputs can be built from the source. +Bazel builds software from source code organized in a directory tree called a +workspace. Source files in the workspace are organized in a nested hierarchy of +packages, where each package is a directory that contains a set of related +source files and one `BUILD` file. The `BUILD` file specifies what software +outputs can be built from the source. ## Workspace {:#workspace} A _workspace_ is a directory tree on your filesystem that contains the source files for the software you want to build. Each workspace has a text file named -`WORKSPACE` which may be empty, or may contain references to -[external dependencies](/docs/external) required to build the outputs. +`WORKSPACE` which may be empty, or may contain references to [external +dependencies](/docs/external) required to build the outputs. Directories containing a file called `WORKSPACE` are considered the root of a -workspace. Therefore, Bazel ignores any directory trees in a workspace rooted -at a subdirectory containing a `WORKSPACE` file, as they form another workspace. +workspace. Therefore, Bazel ignores any directory trees in a workspace rooted at +a subdirectory containing a `WORKSPACE` file, as they form another workspace. -Bazel also supports `WORKSPACE.bazel` file as an alias of `WORKSPACE` file. -If both files exist, `WORKSPACE.bazel` is used. +Bazel also supports `WORKSPACE.bazel` file as an alias of `WORKSPACE` file. If +both files exist, `WORKSPACE.bazel` is used. ### Repositories {:#repositories} Code is organized in _repositories_. The directory containing the `WORKSPACE` file is the root of the main repository, also called `@`. Other, (external) -repositories are defined in the `WORKSPACE` file using workspace rules. +repositories are defined in the `WORKSPACE` file using workspace rules, or +generated from modules and extensions in the Bzlmod system. See [external +dependencies overview](/external/overview) for more information. -The workspace rules bundled with Bazel are documented in the -[Workspace Rules](/reference/be/workspace) section in the -[Build Encyclopedia](/reference/be/overview) and the documentation on -[embedded Starlark repository rules](/rules/lib/repo/index). +The workspace rules bundled with Bazel are documented in the [Workspace +Rules](/reference/be/workspace) section in the [Build +Encyclopedia](/reference/be/overview) and the documentation on [embedded +Starlark repository rules](/rules/lib/repo/index). As external repositories are repositories themselves, they often contain a `WORKSPACE` file as well. However, these additional `WORKSPACE` files are -ignored by Bazel. In particular, repositories depended upon transitively are -not added automatically. +ignored by Bazel. In particular, repositories depended upon transitively are not +added automatically. ## Packages {:#packages} The primary unit of code organization in a repository is the _package_. A -package is a collection of related files and a specification of how they -can be used to produce output artifacts. +package is a collection of related files and a specification of how they can be +used to produce output artifacts. -A package is defined as a directory containing a file named `BUILD` -(or `BUILD.bazel`). A package includes all files in its directory, plus -all subdirectories beneath it, except those which themselves contain a -`BUILD` file. From this definition, no file or directory may be a part of -two different packages. +A package is defined as a directory containing a file named `BUILD` (or +`BUILD.bazel`). A package includes all files in its directory, plus all +subdirectories beneath it, except those which themselves contain a `BUILD` file. +From this definition, no file or directory may be a part of two different +packages. -For example, in the following directory tree -there are two packages, `my/app`, and the subpackage `my/app/tests`. -Note that `my/app/data` is not a package, but a directory -belonging to package `my/app`. +For example, in the following directory tree there are two packages, `my/app`, +and the subpackage `my/app/tests`. Note that `my/app/data` is not a package, but +a directory belonging to package `my/app`. ``` src/my/app/BUILD @@ -68,52 +69,47 @@ src/my/app/tests/test.cc ## Targets {:#targets} A package is a container of _targets_, which are defined in the package's -`BUILD` file. Most targets are one of two principal kinds, _files_ and _rules_. - -Files are further divided into two kinds. _Source files_ are usually -written by the efforts of people, and checked in to the repository. -_Generated files_, sometimes called derived files or output files, -are not checked in, but are generated from source files. - -The second kind of target is declared with a _rule_. Each rule -instance specifies the relationship between a set of input and a set of -output files. The inputs to a rule may be source files, but they also -may be the outputs of other rules. - -Whether the input to a rule is a source file or a generated file is -in most cases immaterial; what matters is only the contents of that -file. This fact makes it easy to replace a complex source file with -a generated file produced by a rule, such as happens when the burden -of manually maintaining a highly structured file becomes too -tiresome, and someone writes a program to derive it. No change is -required to the consumers of that file. Conversely, a generated -file may easily be replaced by a source file with only local -changes. - -The inputs to a rule may also include _other rules_. The -precise meaning of such relationships is often quite complex and -language- or rule-dependent, but intuitively it is simple: a C++ -library rule A might have another C++ library rule B for an input. -The effect of this dependency is that B's header files are -available to A during compilation, B's symbols are available to A -during linking, and B's runtime data is available to A during -execution. - -An invariant of all rules is that the files generated by a rule -always belong to the same package as the rule itself; it is not -possible to generate files into another package. It is not uncommon -for a rule's inputs to come from another package, though. - -Package groups are sets of packages whose purpose is to limit accessibility -of certain rules. Package groups are defined by the `package_group` function. -They have three properties: the list of packages they contain, their name, and -other package groups they include. The only allowed ways to refer to them are -from the `visibility` attribute of rules or from the `default_visibility` -attribute of the `package` function; they do not generate or consume files. -For more information, refer to the -[`package_group` documentation](/reference/be/functions#package_group). - +`BUILD` file. Most targets are one of two principal kinds, _files_ and _rules_. + +Files are further divided into two kinds. _Source files_ are usually written by +the efforts of people, and checked in to the repository. _Generated files_, +sometimes called derived files or output files, are not checked in, but are +generated from source files. + +The second kind of target is declared with a _rule_. Each rule instance +specifies the relationship between a set of input and a set of output files. The +inputs to a rule may be source files, but they also may be the outputs of other +rules. + +Whether the input to a rule is a source file or a generated file is in most +cases immaterial; what matters is only the contents of that file. This fact +makes it easy to replace a complex source file with a generated file produced by +a rule, such as happens when the burden of manually maintaining a highly +structured file becomes too tiresome, and someone writes a program to derive it. +No change is required to the consumers of that file. Conversely, a generated +file may easily be replaced by a source file with only local changes. + +The inputs to a rule may also include _other rules_. The precise meaning of such +relationships is often quite complex and language- or rule-dependent, but +intuitively it is simple: a C++ library rule A might have another C++ library +rule B for an input. The effect of this dependency is that B's header files are +available to A during compilation, B's symbols are available to A during +linking, and B's runtime data is available to A during execution. + +An invariant of all rules is that the files generated by a rule always belong to +the same package as the rule itself; it is not possible to generate files into +another package. It is not uncommon for a rule's inputs to come from another +package, though. + +Package groups are sets of packages whose purpose is to limit accessibility of +certain rules. Package groups are defined by the `package_group` function. They +have three properties: the list of packages they contain, their name, and other +package groups they include. The only allowed ways to refer to them are from the +`visibility` attribute of rules or from the `default_visibility` attribute of +the `package` function; they do not generate or consume files. For more +information, refer to the [`package_group` +documentation](/reference/be/functions#package_group). Labels - + \ No newline at end of file diff --git a/site/en/docs/external.md b/site/en/docs/external.md deleted file mode 100644 index 74d4b78260c16d..00000000000000 --- a/site/en/docs/external.md +++ /dev/null @@ -1,368 +0,0 @@ -Project: /_project.yaml -Book: /_book.yaml - -# Working with External Dependencies - -Bazel can depend on targets from other projects. Dependencies from these other -projects are called _external dependencies_. - -Note: Bazel 5.0 and newer has a new external dependency system, codenamed -"Bzlmod", which renders a lot of the content on this page obsolete. See [Bzlmod -user guide](/docs/bzlmod) for more information. - -The `WORKSPACE` file (or `WORKSPACE.bazel` file) in the -[workspace directory](/concepts/build-ref#workspace) -tells Bazel how to get other projects' sources. These other projects can -contain one or more `BUILD` files with their own targets. `BUILD` files within -the main project can depend on these external targets by using their name from -the `WORKSPACE` file. - -For example, suppose there are two projects on a system: - -``` -/ - home/ - user/ - project1/ - WORKSPACE - BUILD - srcs/ - ... - project2/ - WORKSPACE - BUILD - my-libs/ -``` - -If `project1` wanted to depend on a target, `:foo`, defined in -`/home/user/project2/BUILD`, it could specify that a repository named -`project2` could be found at `/home/user/project2`. Then targets in -`/home/user/project1/BUILD` could depend on `@project2//:foo`. - -The `WORKSPACE` file allows users to depend on targets from other parts of the -filesystem or downloaded from the internet. It uses the same syntax as `BUILD` -files, but allows a different set of rules called _repository rules_ (sometimes -also known as _workspace rules_). Bazel comes with a few [built-in repository -rules](/reference/be/workspace) and a set of [embedded Starlark repository -rules](/rules/lib/repo/index). Users can also write [custom repository -rules](/rules/repository_rules) to get more complex behavior. - -## Supported types of external dependencies {:#types} - -A few basic types of external dependencies can be used: - -- [Dependencies on other Bazel projects](#bazel-projects) -- [Dependencies on non-Bazel projects](#non-bazel-projects) -- [Dependencies on external packages](#external-packages) - -### Depending on other Bazel projects {:#bazel-projects} - -If you want to use targets from a second Bazel project, you can -use -[`local_repository`](/reference/be/workspace#local_repository), -[`git_repository`](/rules/lib/repo/git#git_repository) -or [`http_archive`](/rules/lib/repo/http#http_archive) -to symlink it from the local filesystem, reference a git repository or download -it (respectively). - -For example, suppose you are working on a project, `my-project/`, and you want -to depend on targets from your coworker's project, `coworkers-project/`. Both -projects use Bazel, so you can add your coworker's project as an external -dependency and then use any targets your coworker has defined from your own -BUILD files. You would add the following to `my_project/WORKSPACE`: - -```python -local_repository( - name = "coworkers_project", - path = "/path/to/coworkers-project", -) -``` - -If your coworker has a target `//foo:bar`, your project can refer to it as -`@coworkers_project//foo:bar`. External project names must be -[valid workspace names](/rules/lib/globals#workspace). - -### Depending on non-Bazel projects {:#non-bazel-projects} - -Rules prefixed with `new_`, such as -[`new_local_repository`](/reference/be/workspace#new_local_repository), -allow you to create targets from projects that do not use Bazel. - -For example, suppose you are working on a project, `my-project/`, and you want -to depend on your coworker's project, `coworkers-project/`. Your coworker's -project uses `make` to build, but you'd like to depend on one of the .so files -it generates. To do so, add the following to `my_project/WORKSPACE`: - -```python -new_local_repository( - name = "coworkers_project", - path = "/path/to/coworkers-project", - build_file = "coworker.BUILD", -) -``` - -`build_file` specifies a `BUILD` file to overlay on the existing project, for -example: - -```python -cc_library( - name = "some-lib", - srcs = glob(["**"]), - visibility = ["//visibility:public"], -) -``` - -You can then depend on `@coworkers_project//:some-lib` from your project's -`BUILD` files. - -### Depending on external packages {:#external-packages} - -#### Maven artifacts and repositories {:#maven-repositories} - -Use the ruleset [`rules_jvm_external`](https://github.com/bazelbuild/rules_jvm_external){: .external} -to download artifacts from Maven repositories and make them available as Java -dependencies. - -## Fetching dependencies {:#fetching-dependencies} - -By default, external dependencies are fetched as needed during `bazel build`. If -you would like to prefetch the dependencies needed for a specific set of targets, use -[`bazel fetch`](/reference/command-line-reference#commands). -To unconditionally fetch all external dependencies, use -[`bazel sync`](/reference/command-line-reference#commands). -As fetched repositories are [stored in the output base](#layout), fetching -happens per workspace. - -## Shadowing dependencies {:#shadowing-dependencies} - -Whenever possible, it is recommended to have a single version policy in your -project. This is required for dependencies that you compile against and end up -in your final binary. But for cases where this isn't true, it is possible to -shadow dependencies. Consider the following scenario: - -myproject/WORKSPACE - -```python -workspace(name = "myproject") - -local_repository( - name = "A", - path = "../A", -) -local_repository( - name = "B", - path = "../B", -) -``` - -A/WORKSPACE - -```python -workspace(name = "A") - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -http_archive( - name = "testrunner", - urls = ["https://github.com/testrunner/v1.zip"], - sha256 = "...", -) -``` - -B/WORKSPACE - -```python -workspace(name = "B") - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -http_archive( - name = "testrunner", - urls = ["https://github.com/testrunner/v2.zip"], - sha256 = "..." -) -``` - -Both dependencies `A` and `B` depend on `testrunner`, but they depend on -different versions of `testrunner`. There is no reason for these test runners to -not peacefully coexist within `myproject`, however they will clash with each -other since they have the same name. To declare both dependencies, -update myproject/WORKSPACE: - -```python -workspace(name = "myproject") - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -http_archive( - name = "testrunner-v1", - urls = ["https://github.com/testrunner/v1.zip"], - sha256 = "..." -) -http_archive( - name = "testrunner-v2", - urls = ["https://github.com/testrunner/v2.zip"], - sha256 = "..." -) -local_repository( - name = "A", - path = "../A", - repo_mapping = {"@testrunner" : "@testrunner-v1"} -) -local_repository( - name = "B", - path = "../B", - repo_mapping = {"@testrunner" : "@testrunner-v2"} -) -``` - -This mechanism can also be used to join diamonds. For example if `A` and `B` -had the same dependency but call it by different names, those dependencies can -be joined in myproject/WORKSPACE. - -## Overriding repositories from the command line {:#overriding-repositories} - -To override a declared repository with a local repository from the command line, -use the -[`--override_repository`](/reference/command-line-reference#flag--override_repository) -flag. Using this flag changes the contents of external repositories without -changing your source code. - -For example, to override `@foo` to the local directory `/path/to/local/foo`, -pass the `--override_repository=foo=/path/to/local/foo` flag. - -Some of the use cases include: - -* Debugging issues. For example, you can override a `http_archive` repository - to a local directory where you can make changes more easily. -* Vendoring. If you are in an environment where you cannot make network calls, - override the network-based repository rules to point to local directories - instead. - -## Using proxies {:#using-proxies} - -Bazel will pick up proxy addresses from the `HTTPS_PROXY` and `HTTP_PROXY` -environment variables and use these to download HTTP/HTTPS files (if specified). - -## Support for IPv6 {:#support-for-ipv6} - -On IPv6-only machines, Bazel will be able to download dependencies with -no changes. On dual-stack IPv4/IPv6 machines, however, Bazel follows the same -convention as Java: if IPv4 is enabled, IPv4 is preferred. In some situations, -for example when IPv4 network is unable to resolve/reach external addresses, -this can cause `Network unreachable` exceptions and build failures. -In these cases, you can override Bazel's behavior to prefer IPv6 -by using [`java.net.preferIPv6Addresses=true` system property](https://docs.oracle.com/javase/8/docs/api/java/net/doc-files/net-properties.html){: .external}. -Specifically: - -* Use `--host_jvm_args=-Djava.net.preferIPv6Addresses=true` - [startup option](/docs/user-manual#startup-options), - for example by adding the following line in your - [`.bazelrc` file](/docs/bazelrc): - - `startup --host_jvm_args=-Djava.net.preferIPv6Addresses=true` - -* If you are running Java build targets that need to connect to the internet - as well (integration tests sometimes needs that), also use - `--jvmopt=-Djava.net.preferIPv6Addresses=true` - [tool flag](/docs/user-manual#jvmopt), for example by having the - following line in your [`.bazelrc` file](/docs/bazelrc): - - `build --jvmopt=-Djava.net.preferIPv6Addresses` - -* If you are using - [rules_jvm_external](https://github.com/bazelbuild/rules_jvm_external){: .external}, - for example, for dependency version resolution, also add - `-Djava.net.preferIPv6Addresses=true` to the `COURSIER_OPTS` - environment variable to [provide JVM options for Coursier](https://github.com/bazelbuild/rules_jvm_external#provide-jvm-options-for-coursier-with-coursier_opts){: .external} - -## Transitive dependencies {:#transitive-dependencies} - -Bazel only reads dependencies listed in your `WORKSPACE` file. If your project -(`A`) depends on another project (`B`) which lists a dependency on a third -project (`C`) in its `WORKSPACE` file, you'll have to add both `B` -and `C` to your project's `WORKSPACE` file. This requirement can balloon the -`WORKSPACE` file size, but limits the chances of having one library -include `C` at version 1.0 and another include `C` at 2.0. - -## Caching of external dependencies {:#caching-external-dependencies} - -By default, Bazel will only re-download external dependencies if their -definition changes. Changes to files referenced in the definition (such as patches -or `BUILD` files) are also taken into account by bazel. - -To force a re-download, use `bazel sync`. - -## Layout {:#layout} - -External dependencies are all downloaded to a directory under the subdirectory -`external` in the [output base](/remote/output-directories). In case of a -[local repository](/reference/be/workspace#local_repository), a symlink is created -there instead of creating a new directory. -You can see the `external` directory by running: - -```posix-terminal -ls $(bazel info output_base)/external -``` - -Note that running `bazel clean` will not actually delete the external -directory. To remove all external artifacts, use `bazel clean --expunge`. - -## Offline builds {:#offline-builds} - -It is sometimes desirable or necessary to run a build in an offline fashion. For -simple use cases, such as traveling on an airplane, -[prefetching](#fetching-dependencies) the needed -repositories with `bazel fetch` or `bazel sync` can be enough; moreover, the -using the option `--nofetch`, fetching of further repositories can be disabled -during the build. - -For true offline builds, where the providing of the needed files is to be done -by an entity different from bazel, bazel supports the option -`--distdir`. Whenever a repository rule asks bazel to fetch a file via -[`ctx.download`](/rules/lib/repository_ctx#download) or -[`ctx.download_and_extract`](/rules/lib/repository_ctx#download_and_extract) -and provides a hash sum of the file -needed, bazel will first look into the directories specified by that option for -a file matching the basename of the first URL provided, and use that local copy -if the hash matches. - -Bazel itself uses this technique to bootstrap offline from the [distribution -artifact](https://github.com/bazelbuild/bazel-website/blob/master/designs/_posts/2016-10-11-distribution-artifact.md). -It does so by [collecting all the needed external -dependencies](https://github.com/bazelbuild/bazel/blob/5cfa0303d6ac3b5bd031ff60272ce80a704af8c2/WORKSPACE#L116){: .external} -in an internal -[`distdir_tar`](https://github.com/bazelbuild/bazel/blob/5cfa0303d6ac3b5bd031ff60272ce80a704af8c2/distdir.bzl#L44){: .external}. - -However, bazel allows the execution of arbitrary commands in repository rules, -without knowing if they call out to the network. Therefore, bazel has no option -to enforce builds being fully offline. So testing if a build works correctly -offline requires external blocking of the network, as bazel does in its -bootstrap test. - -## Best practices {:#best-practices} - -### Repository rules {:#repository-rules} - -A repository rule should generally be responsible for: - -- Detecting system settings and writing them to files. -- Finding resources elsewhere on the system. -- Downloading resources from URLs. -- Generating or symlinking BUILD files into the external repository directory. - -Avoid using `repository_ctx.execute` when possible. For example, when using a non-Bazel C++ -library that has a build using Make, it is preferable to use `repository_ctx.download()` and then -write a BUILD file that builds it, instead of running `ctx.execute(["make"])`. - -Prefer [`http_archive`](/rules/lib/repo/http#http_archive) to `git_repository` and -`new_git_repository`. The reasons are: - -* Git repository rules depend on system `git(1)` whereas the HTTP downloader is built - into Bazel and has no system dependencies. -* `http_archive` supports a list of `urls` as mirrors, and `git_repository` supports only - a single `remote`. -* `http_archive` works with the [repository cache](/docs/build#repository-cache), but not - `git_repository`. See - [#5116](https://github.com/bazelbuild/bazel/issues/5116){: .external} for more information. - -Do not use `bind()`. See "[Consider removing -bind](https://github.com/bazelbuild/bazel/issues/1952){: .external}" for a long -discussion of its issues and alternatives. diff --git a/site/en/external/advanced.md b/site/en/external/advanced.md new file mode 100644 index 00000000000000..6e4f472f6fd27c --- /dev/null +++ b/site/en/external/advanced.md @@ -0,0 +1,184 @@ +Project: /_project.yaml +Book: /_book.yaml + +# Advanced topics on external dependencies + +{% include "_buttons.html" %} + +## Shadowing dependencies in WORKSPACE + +Note: This section applies to the [WORKSPACE +system](/external/overview#workspace-system) only. For +[Bzlmod](/external/overview#bzlmod), use a [multiple-version +override](/external/module#multiple-version_override). + +Whenever possible, have a single version policy in your project, which is +required for dependencies that you compile against and end up in your final +binary. For other cases, you can shadow dependencies: + +myproject/WORKSPACE + +```python +workspace(name = "myproject") + +local_repository( + name = "A", + path = "../A", +) +local_repository( + name = "B", + path = "../B", +) +``` + +A/WORKSPACE + +```python +workspace(name = "A") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +http_archive( + name = "testrunner", + urls = ["https://github.com/testrunner/v1.zip"], + sha256 = "...", +) +``` + +B/WORKSPACE {# This is not a buganizer link okay?? #} + +```python +workspace(name = "B") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +http_archive( + name = "testrunner", + urls = ["https://github.com/testrunner/v2.zip"], + sha256 = "..." +) +``` + +Both dependencies `A` and `B` depend on different versions of `testrunner`. +Include both in `myproject` without conflict by giving them distinct names in +`myproject/WORKSPACE`: + +```python +workspace(name = "myproject") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +http_archive( + name = "testrunner-v1", + urls = ["https://github.com/testrunner/v1.zip"], + sha256 = "..." +) +http_archive( + name = "testrunner-v2", + urls = ["https://github.com/testrunner/v2.zip"], + sha256 = "..." +) +local_repository( + name = "A", + path = "../A", + repo_mapping = {"@testrunner" : "@testrunner-v1"} +) +local_repository( + name = "B", + path = "../B", + repo_mapping = {"@testrunner" : "@testrunner-v2"} +) +``` + +You can also use this mechanism to join diamonds. For example, if `A` and `B` +have the same dependency but call it by different names, join those dependencies +in `myproject/WORKSPACE`. + +## Overriding repositories from the command line {:#overriding-repositories} + +To override a declared repository with a local repository from the command line, +use the +[`--override_repository`](/reference/command-line-reference#flag--override_repository) +flag. Using this flag changes the contents of external repositories without +changing your source code. + +For example, to override `@foo` to the local directory `/path/to/local/foo`, +pass the `--override_repository=foo=/path/to/local/foo` flag. + +Use cases include: + +* Debugging issues. For example, to override an `http_archive` repository to a + local directory where you can make changes more easily. +* Vendoring. If you are in an environment where you cannot make network calls, + override the network-based repository rules to point to local directories + instead. + +Note: With [Bzlmod](/external/overview#bzlmod), remember to use canonical repo +names here. Alternatively, use the +[`--override_module`](/reference/command-line-reference#flag--override_module) +flag to override a module to a local directory, similar to the +[`local_path_override`](/rules/lib/globals#local_path_override) directive in +`MODULE.bazel`. + +## Using proxies + +Bazel picks up proxy addresses from the `HTTPS_PROXY` and `HTTP_PROXY` +environment variables and uses these to download `HTTP` and `HTTPS` files (if +specified). + +## Support for IPv6 + +On IPv6-only machines, Bazel can download dependencies with no changes. However, +on dual-stack IPv4/IPv6 machines Bazel follows the same convention as Java, +preferring IPv4 if enabled. In some situations, for example when the IPv4 +network cannot resolve/reach external addresses, this can cause `Network +unreachable` exceptions and build failures. In these cases, you can override +Bazel's behavior to prefer IPv6 by using the +[`java.net.preferIPv6Addresses=true` system +property](https://docs.oracle.com/javase/8/docs/api/java/net/doc-files/net-properties.html){: .external}. +Specifically: + +* Use `--host_jvm_args=-Djava.net.preferIPv6Addresses=true` [startup + option](/docs/user-manual#startup-options), for example by adding the + following line in your [`.bazelrc` file](/run/bazelrc): + + `startup --host_jvm_args=-Djava.net.preferIPv6Addresses=true` + +* When running Java build targets that need to connect to the internet (such + as for integration tests), use the + `--jvmopt=-Djava.net.preferIPv6Addresses=true` [tool + flag](/docs/user-manual#jvmopt). For example, include in your [`.bazelrc` + file](/run/bazelrc): + + `build --jvmopt=-Djava.net.preferIPv6Addresses` + +* If you are using [`rules_jvm_external`](https://github.com/bazelbuild/rules_jvm_external){: .external} + for dependency version resolution, also add + `-Djava.net.preferIPv6Addresses=true` to the `COURSIER_OPTS` environment + variable to [provide JVM options for + Coursier](https://github.com/bazelbuild/rules_jvm_external#provide-jvm-options-for-coursier-with-coursier_opts){: .external}. + +## Offline builds + +Sometimes you may wish to run a build offline, such as when traveling on an +airplane. For such simple use cases, prefetch the needed repositories with +`bazel fetch` or `bazel sync`. To disable fetching further repositories during +the build, use the option `--nofetch`. + +For true offline builds, where a different entity supplies all needed files, +Bazel supports the option `--distdir`. This flag tells Bazel to look first into +the directories specified by that option when a repository rule asks Bazel to +fetch a file with [`ctx.download`](/rules/lib/repository_ctx#download) or +[`ctx.download_and_extract`](/rules/lib/repository_ctx#download_and_extract). By +providing a hash sum of the file needed, Bazel looks for a file matching the +basename of the first URL, and uses the local copy if the hash matches. + +Bazel itself uses this technique to bootstrap offline from the [distribution +artifact](https://github.com/bazelbuild/bazel-website/blob/master/designs/_posts/2016-10-11-distribution-artifact.md). +It does so by [collecting all the needed external +dependencies](https://github.com/bazelbuild/bazel/blob/5cfa0303d6ac3b5bd031ff60272ce80a704af8c2/WORKSPACE#L116){: .external} +in an internal +[`distdir_tar`](https://github.com/bazelbuild/bazel/blob/5cfa0303d6ac3b5bd031ff60272ce80a704af8c2/distdir.bzl#L44){: .external}. + +Bazel allows execution of arbitrary commands in repository rules without knowing +if they call out to the network, and so cannot enforce fully offline builds. To +test if a build works correctly offline, manually block off the network (as +Bazel does in its [bootstrap +test](https://cs.opensource.google/bazel/bazel/+/master:src/test/shell/bazel/BUILD;l=1073;drc=88c426e73cc0eb0a41c0d7995e36acd94e7c9a48){: .external}). \ No newline at end of file diff --git a/site/en/external/extension.md b/site/en/external/extension.md new file mode 100644 index 00000000000000..6a9a99498237d0 --- /dev/null +++ b/site/en/external/extension.md @@ -0,0 +1,177 @@ +Project: /_project.yaml +Book: /_book.yaml + +# Module extensions + +{% include "_buttons.html" %} + +Module extensions allow users to extend the module system by reading input data +from modules across the dependency graph, performing necessary logic to resolve +dependencies, and finally creating repos by calling repo rules. These extensions +have capabilities similar to repo rules, which enables them to perform file I/O, +send network requests, and so on. Among other things, they allow Bazel to +interact with other package management systems while also respecting the +dependency graph built out of Bazel modules. + +You can define module extensions in `.bzl` files, just like repo rules. They're +not invoked directly; rather, each module specifies pieces of data called *tags* +for extensions to read. Bazel runs module resolution before evaluating any +extensions. The extension reads all the tags belonging to it across the entire +dependency graph. + +## Extension usage + +Extensions are hosted in Bazel modules themselves. To use an extension in a +module, first add a `bazel_dep` on the module hosting the extension, and then +call the [`use_extension`](/rules/lib/globals#use_extension) built-in function +to bring it into scope. Consider the following example — a snippet from a +`MODULE.bazel` file to use the "maven" extension defined in the +[`rules_jvm_external`](https://github.com/bazelbuild/rules_jvm_external){:.external} +module: + +```python +bazel_dep(name = "rules_jvm_external", version = "4.5") +maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven") +``` + +This binds the return value of `use_extension` to a variable, which allows the +user to use dot-syntax to specify tags for the extension. The tags must follow +the schema defined by the corresponding *tag classes* specified in the +[extension definition](#extension_definition). For an example specifying some +`maven.install` and `maven.artifact` tags: + +```python +maven.install(artifacts = ["org.junit:junit:4.13.2"]) +maven.artifact(group = "com.google.guava", + artifact = "guava", + version = "27.0-jre", + exclusions = ["com.google.j2objc:j2objc-annotations"]) +``` + +Use the [`use_repo`](/rules/lib/globals#use_repo) directive to bring repos +generated by the extension into the scope of the current module. + +```python +use_repo(maven, "maven") +``` + +Repos generated by an extension are part of its API. In this example, the +"maven" module extension promises to generate a repo called `maven`. With the +declaration above, the extension properly resolves labels such as +`@maven//:org_junit_junit` to point to the repo generated by the "maven" +extension. + +## Extension definition + +You can define module extensions similarly to repo rules, using the +[`module_extension`](/rules/lib/globals#module_extension) function. However, +while repo rules have a number of attributes, module extensions have +[`tag_class`es](/rules/lib/globals#tag_class), each of which has a number of +attributes. The tag classes define schemas for tags used by this extension. For +example, the "maven" extension above might be defined like this: + +```python +# @rules_jvm_external//:extensions.bzl + +_install = tag_class(attrs = {"artifacts": attr.string_list(), ...}) +_artifact = tag_class(attrs = {"group": attr.string(), "artifact": attr.string(), ...}) +maven = module_extension( + implementation = _maven_impl, + tag_classes = {"install": _install, "artifact": _artifact}, +) +``` + +These declarations show that `maven.install` and `maven.artifact` tags can be +specified using the specified attribute schema. + +The implementation function of module extensions are similar to those of repo +rules, except that they get a [`module_ctx`](/rules/lib/module_ctx) object, +which grants access to all modules using the extension and all pertinent tags. +The implementation function then calls repo rules to generate repos. + +```python +# @rules_jvm_external//:extensions.bzl + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") # a repo rule +def _maven_impl(ctx): + # This is a fake implementation for demonstration purposes only + + # collect artifacts from across the dependency graph + artifacts = [] + for mod in ctx.modules: + for install in mod.tags.install: + artifacts += install.artifacts + artifacts += [_to_artifact(artifact) for artifact in mod.tags.artifact] + + # call out to the coursier CLI tool to resolve dependencies + output = ctx.execute(["coursier", "resolve", artifacts]) + repo_attrs = _process_coursier_output(output) + + # call repo rules to generate repos + for attrs in repo_attrs: + http_file(**attrs) + _generate_hub_repo(name = "maven", repo_attrs) +``` + +### Extension identity + +Module extensions are identified by the name and the `.bzl` file that appears +in the call to `use_extension`. In the following example, the extension `maven` +is identified by the `.bzl` file `@rules_jvm_external//:extension.bzl` and the +name `maven`: + +```python +maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven") +``` + +Re-exporting an extension from a different `.bzl` file gives it a new identity +and if both versions of the extension are used in the transitive module graph, +then they will be evaluated separately and will only see the tags associated +with that particular identity. + +As an extension author you should make sure that users will only use your +module extension from one single `.bzl` file. + +## Repository names and visibility + +Repos generated by extensions have canonical names in the form of `{{ "" +}}module_repo_canonical_name{{ "" }}~{{ "" }}extension_name{{ +"" }}~{{ "" }}repo_name{{ "" }}`. For extensions hosted in the +root module, the `{{ "" }}module_repo_canonical_name{{ "" }}` part is +replaced with the string `_main`. Note that the canonical name format is not an +API you should depend on — it's subject to change at any time. + +This naming policy means that each extension has its own "repo namespace"; two +distinct extensions can each define a repo with the same name without risking +any clashes. It also means that `repository_ctx.name` reports the canonical name +of the repo, which is *not* the same as the name specified in the repo rule +call. + +Taking repos generated by module extensions into consideration, there are +several repo visibility rules: + +* A Bazel module repo can see all repos introduced in its `MODULE.bazel` file + via [`bazel_dep`](/rules/lib/globals#bazel_dep) and + [`use_repo`](/rules/lib/globals#use_repo). +* A repo generated by a module extension can see all repos visible to the + module that hosts the extension, *plus* all other repos generated by the + same module extension (using the names specified in the repo rule calls as + their apparent names). + * This might result in a conflict. If the module repo can see a repo with + the apparent name `foo`, and the extension generates a repo with the + specified name `foo`, then for all repos generated by that extension + `foo` refers to the former. + +## Best practices + +This section describes best practices when writing extensions so they are +straightforward to use, maintainable, and adapt well to changes over time. + +### Put each extension in a separate file + +When extensions are in a different files, it allows one extension to load +repositories generated by another extension. Even if you don't use this +functionality, it's best to put them in separate files in case you need it +later. This is because the extension's identify is based on its file, so moving +the extension into another file later changes your public API and is a backwards +incompatible change for your users. \ No newline at end of file diff --git a/site/en/external/images/mod_exampleBefore.svg b/site/en/external/images/mod_exampleBefore.svg new file mode 100644 index 00000000000000..66f01303f18e36 --- /dev/null +++ b/site/en/external/images/mod_exampleBefore.svg @@ -0,0 +1,175 @@ + + + + + + +mygraph + + + +<root> + +<root> (my_project@1.0) + + + +bazel_skylib@1.1.1 + +bazel_skylib@1.1.1 + + + +<root>->bazel_skylib@1.1.1 + + + + + +bazel_skylib@1.2.0 + +bazel_skylib@1.2.0 + + + +<root>->bazel_skylib@1.2.0 + + + + + +rules_java@5.0.0 + +rules_java@5.0.0 + + + +<root>->rules_java@5.0.0 + + + + + +stardoc@0.5.0 + +stardoc@0.5.0 + + + +<root>->stardoc@0.5.0 + + + + + +platforms@0.0.4 + +platforms@0.0.4 + + + +bazel_skylib@1.1.1->platforms@0.0.4 + + + + + +bazel_skylib@1.2.0->platforms@0.0.4 + + + + + +rules_java@5.0.0->platforms@0.0.4 + + + + + +rules_cc@0.0.1 + +rules_cc@0.0.1 + + + +rules_java@5.0.0->rules_cc@0.0.1 + + + + + +rules_proto@4.0.0 + +rules_proto@4.0.0 + + + +rules_java@5.0.0->rules_proto@4.0.0 + + + + + +bazel_skylib@1.0.3 + +bazel_skylib@1.0.3 + + + +stardoc@0.5.0->bazel_skylib@1.0.3 + + + + + +rules_java@4.0.0 + +rules_java@4.0.0 + + + +stardoc@0.5.0->rules_java@4.0.0 + + + + + +rules_cc@0.0.1->platforms@0.0.4 + + + + + +rules_cc@0.0.1->bazel_skylib@1.0.3 + + + + + +rules_proto@4.0.0->rules_cc@0.0.1 + + + + + +rules_proto@4.0.0->bazel_skylib@1.0.3 + + + + + +bazel_skylib@1.0.3->platforms@0.0.4 + + + + + +rules_java@4.0.0->bazel_skylib@1.0.3 + + + + + \ No newline at end of file diff --git a/site/en/external/images/mod_exampleResolved.svg b/site/en/external/images/mod_exampleResolved.svg new file mode 100644 index 00000000000000..224b694c0c5c8c --- /dev/null +++ b/site/en/external/images/mod_exampleResolved.svg @@ -0,0 +1,151 @@ + + + + + + +mygraph + + + +<root> + +<root> (my_project@1.0) + + + +bazel_skylib@1.1.1 + +bazel_skylib@1.1.1 + + + +<root>->bazel_skylib@1.1.1 + + + + + +bazel_skylib@1.2.0 + +bazel_skylib@1.2.0 + + + +<root>->bazel_skylib@1.2.0 + + + + + +rules_java@5.0.0 + +rules_java@5.0.0 + + + +<root>->rules_java@5.0.0 + + + + + +stardoc@0.5.0 + +stardoc@0.5.0 + + + +<root>->stardoc@0.5.0 + + + + + +platforms@0.0.4 + +platforms@0.0.4 + + + +bazel_skylib@1.1.1->platforms@0.0.4 + + + + + +bazel_skylib@1.2.0->platforms@0.0.4 + + + + + +rules_java@5.0.0->platforms@0.0.4 + + + + + +rules_cc@0.0.1 + +rules_cc@0.0.1 + + + +rules_java@5.0.0->rules_cc@0.0.1 + + + + + +rules_proto@4.0.0 + +rules_proto@4.0.0 + + + +rules_java@5.0.0->rules_proto@4.0.0 + + + + + +stardoc@0.5.0->bazel_skylib@1.1.1 + + + + + +stardoc@0.5.0->rules_java@5.0.0 + + + + + +rules_cc@0.0.1->bazel_skylib@1.1.1 + + + + + +rules_cc@0.0.1->platforms@0.0.4 + + + + + +rules_proto@4.0.0->bazel_skylib@1.1.1 + + + + + +rules_proto@4.0.0->rules_cc@0.0.1 + + + + + \ No newline at end of file diff --git a/site/en/external/migration.md b/site/en/external/migration.md new file mode 100644 index 00000000000000..de9a5ba65155ee --- /dev/null +++ b/site/en/external/migration.md @@ -0,0 +1,803 @@ +Project: /_project.yaml +Book: /_book.yaml +keywords: bzlmod + +# Bzlmod Migration Guide + +{% include "_buttons.html" %} + +Due to the [shortcomings of +WORKSPACE](/external/overview#workspace-shortcomings), Bzlmod is going to +replace the legacy WORKSPACE system in future Bazel releases. This guide helps +you migrate your project to Bzlmod and drop WORKSPACE for fetching external +dependencies. + +## WORKSPACE vs Bzlmod {:#workspace-vs-bzlmod} + +Bazel's WORKSPACE and Bzlmod offer similar features with different syntax. This +section explains how to migrate from specific WORKSPACE functionalities to +Bzlmod. + +### Define the root of a Bazel workspace {:#define-root} + +The WORKSPACE file marks the source root of a Bazel project, this responsibility +is replaced by MODULE.bazel in Bazel version 6.3 and later. With Bazel version +prior to 6.3, there should still be a `WORKSPACE` or `WORKSPACE.bazel` file at +your workspace root, maybe with comments like: + +* **WORKSPACE** + + ```python + # This file marks the root of the Bazel workspace. + # See MODULE.bazel for external dependencies setup. + ``` + +### Specify repository name for your workspace {:#specify-repo-name} + +* **WORKSPACE** + + The [`workspace`](/rules/lib/globals/workspace#workspace) function is used + to specify a repository name for your workspace. This allows a target + `//foo:bar` in the workspace to be referenced as `@//foo:bar`. If not specified, the default repository name for your + workspace is `__main__`. + + ```python + ## WORKSPACE + workspace(name = "com_foo_bar") + ``` + +* **Bzlmod** + + It's recommended to reference targets in the same workspace with the + `//foo:bar` syntax without `@`. But if you do need the old syntax + , you can use the module name specified by the + [`module`](/rules/lib/globals/module#module) function as the repository + name. If the module name is different from the needed repository name, you + can use `repo_name` attribute of the + [`module`](/rules/lib/globals/module#module) function to override the + repository name. + + ```python + ## MODULE.bazel + module( + name = "bar", + repo_name = "com_foo_bar", + ) + ``` + +### Fetch external dependencies as Bazel modules {:#fetch-bazel-modules} + +If your dependency is a Bazel project, you should be able to depend on it as a +Bazel module when it also adopts Bzlmod. + +* **WORKSPACE** + + With WORKSPACE, it's common to use the + [`http_archive`](/rules/lib/repo/http#http_archive) or + [`git_repository`](/rules/lib/repo/git#git_repository) repository rules to + download the sources of the Bazel project. + + ```python + ## WORKSPACE + load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + + http_archive( + name = "bazel_skylib", + urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/1.4.2/bazel-skylib-1.4.2.tar.gz"], + sha256 = "66ffd9315665bfaafc96b52278f57c7e2dd09f5ede279ea6d39b2be471e7e3aa", + ) + load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") + bazel_skylib_workspace() + + http_archive( + name = "rules_java", + urls = ["https://github.com/bazelbuild/rules_java/releases/download/6.1.1/rules_java-6.1.1.tar.gz"], + sha256 = "76402a50ae6859d50bd7aed8c1b8ef09dae5c1035bb3ca7d276f7f3ce659818a", + ) + load("@rules_java//java:repositories.bzl", "rules_java_dependencies", "rules_java_toolchains") + rules_java_dependencies() + rules_java_toolchains() + ``` + + As you can see, it's a common pattern that users need to load transitive + dependencies from a macro of the dependency. Assume both `bazel_skylib` and + `rules_java` depends on `platoform`, the exact version of the `platform` + dependency is determined by the order of the macros. + +* **Bzlmod** + + With Bzlmod, as long as the your dependency is available in [Bazel Central + Registry](https://registry.bazel.build) or your custom [Bazel + registry](/external/registry), you can simply depend on it with a + [`bazel_dep`](/rules/lib/globals/module#bazel_dep) directive. + + ```python + ## MODULE.bazel + bazel_dep(name = "bazel_skylib", version = "1.4.2") + bazel_dep(name = "rules_java", version = "6.1.1") + ``` + + Bzlmod resolves Bazel module dependencies transitively using the + [MVS](https://research.swtch.com/vgo-mvs) algorithm. Therefore, the maximal + required version of `platform` is selected automatically. + +### Override a dependency as a Bazel module{:#override-modules} + +As the root module, you can override Bazel module dependencies in different +ways. + +Please read the [overrides](/external/module#overrides) section for more +information. + +You can find some example usages in the +[examples][override-examples] +repository. + +[override-examples]: https://github.com/bazelbuild/examples/blob/main/bzlmod/02-override_bazel_module + +### Fetch external dependencies with module extensions{:#fetch-deps-module-extensions} + +If your dependency is not a Bazel project or not yet available in any Bazel +registry, you can introduce it using [module extensions](/external/extension). + +* **WORKSPACE** + + Download a file using the [`http_file`](/rules/lib/repo/http#http_file) + repository rule. + + ```python + ## WORKSPACE + load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") + + http_file( + name = "data_file", + url = "http://example.com/file", + sha256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ) + ``` + +* **Bzlmod** + + With Bzlmod, you have to move the definition into a `.bzl` file, which also + lets you share the definition between WORKSPACE and Bzlmod during the + migration period. + + ```python + ## repositories.bzl + load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") + def my_data_dependency(): + http_file( + name = "data_file", + url = "http://example.com/file", + sha256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ) + ``` + + Implement a module extension to load the dependencies macro. You can define + it in the same `.bzl` file of the macro, but to keep compatibility with + older Bazel versions, it's better to define it in a separate `.bzl` file. + + ```python + ## extensions.bzl + load("//:repositories.bzl", "my_data_dependency") + def _non_module_dependencies_impl(_ctx): + my_data_dependency() + + non_module_dependencies = module_extension( + implementation = _non_module_dependencies_impl, + ) + ``` + + To make the repository visible to the root project, you should declare the + usages of the module extension and the repository in the MODULE.bazel file. + + ```python + ## MODULE.bazel + non_module_dependencies = use_extension("//:extensions.bzl", "non_module_dependencies") + use_repo(non_module_dependencies, "data_file") + ``` + +### Resolve conflict external dependencies with module extension {:#conflict-deps-module-extension} + +A project can provide a macro that introduces external repositories based on +inputs from its callers. But what if there are multiple callers in the +dependency graph and they cause a conflict? + +Assume the project `foo` provides the following macro which takes `version` as +an argument. + +```python +## repositories.bzl in foo {:#repositories.bzl-foo} +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") +def data_deps(version = "1.0"): + http_file( + name = "data_file", + url = "http://example.com/file-%s" % version, + # Omitting the "sha256" attribute for simplicity + ) +``` + +* **WORKSPACE** + + With WORKSPACE, you can load the macro from `@foo` and specify the version + of the data dependency you need. Assume you have another dependency `@bar`, + which also depends on `@foo` but requires a different version of the data + dependency. + + ```python + ## WORKSPACE + + # Introduce @foo and @bar. + ... + + load("@foo//:repositories.bzl", "data_deps") + data_deps(version = "2.0") + + load("@bar//:repositories.bzl", "bar_deps") + bar_deps() # -> which calls data_deps(version = "3.0") + ``` + + In this case, the end user must carefully adjust the order of macros in the + WORKSPACE to get the version they need. This is one of the biggest pain + points with WORKSPACE since it doesn't really provide a sensible way to + resolve dependencies. + +* **Bzlmod** + + With Bzlmod, the author of project `foo` can use module extension to resolve + conflicts. For example, let's assume it makes sense to always select the + maximal required version of the data dependency among all Bazel modules. + + ```python + ## extensions.bzl in foo + load("//:repositories.bzl", "data_deps") + + data = tag_class(attrs={"version": attr.string()}) + + def _data_deps_extension_impl(module_ctx): + # Select the maximal required version in the dependency graph. + version = "1.0" + for mod in module_ctx.modules: + for data in mod.tags.data: + version = max(version, data.version) + data_deps(version) + + data_deps_extension = module_extension( + implementation = _data_deps_extension_impl, + tag_classes = {"data": data}, + ) + ``` + + ```python + ## MODULE.bazel in bar + bazel_dep(name = "foo", version = "1.0") + + foo_data_deps = use_extension("@foo//:extensions.bzl", "data_deps_extension") + foo_data_deps.data(version = "3.0") + use_repo(foo_data_deps, "data_file") + ``` + + ```python + ## MODULE.bazel in root module + bazel_dep(name = "foo", version = "1.0") + bazel_dep(name = "bar", version = "1.0") + + foo_data_deps = use_extension("@foo//:extensions.bzl", "data_deps_extension") + foo_data_deps.data(version = "2.0") + use_repo(foo_data_deps, "data_file") + ``` + + In this case, the root module requires data version `2.0`, while its + dependency `bar` requires `3.0`. The module extension in `foo` can correctly + resolve this conflict and automatically select version `3.0` for the data + dependency. + +### Integrate third party package manager {:#integrate-package-manager} + +Following the last section, since module extension provides a way to collect +information from the dependency graph, perform custom logic to resolve +dependencies and call repository rules to introduce external repositories, this +provides a great way for rules authors to enhance the rulesets that integrate +package managers for specific languages. + +Please read the [module extensions](/external/extension) page to learn more +about how to use module extensions. + +Here is a list of the rulesets that already adopted Bzlmod to fetch dependencies +from different package managers: + +- [rules_jvm_external](https://github.com/bazelbuild/rules_jvm_external/blob/master/docs/bzlmod.md) +- [rules_go](https://github.com/bazelbuild/rules_go/blob/master/docs/go/core/bzlmod.md) +- [rules_python](https://github.com/bazelbuild/rules_python/blob/main/BZLMOD_SUPPORT.md) + +A minimal example that integrates a pseudo package manager is available at the +[examples][pkg-mgr-example] +repository. + +[pkg-mgr-example]: https://github.com/bazelbuild/examples/tree/main/bzlmod/05-integrate_third_party_package_manager + +### Detect toolchains on the host machine {:#detect-toolchain} + +When Bazel build rules need to detect what toolchains are available on your host +machine, they use repository rules to inspect the host machine and generate +toolchain info as external repositories. + +* **WORKSPACE** + + Given the following repository rule to detect a shell toolchain. + + ```python + ## local_config_sh.bzl + def _sh_config_rule_impl(repository_ctx): + sh_path = get_sh_path_from_env("SH_BIN_PATH") + + if not sh_path: + sh_path = detect_sh_from_path() + + if not sh_path: + sh_path = "/shell/binary/not/found" + + repository_ctx.file("BUILD", """ + load("@bazel_tools//tools/sh:sh_toolchain.bzl", "sh_toolchain") + sh_toolchain( + name = "local_sh", + path = "{sh_path}", + visibility = ["//visibility:public"], + ) + toolchain( + name = "local_sh_toolchain", + toolchain = ":local_sh", + toolchain_type = "@bazel_tools//tools/sh:toolchain_type", + ) + """.format(sh_path = sh_path)) + + sh_config_rule = repository_rule( + environ = ["SH_BIN_PATH"], + local = True, + implementation = _sh_config_rule_impl, + ) + ``` + + You can load the repository rule in WORKSPACE. + + ```python + ## WORKSPACE + load("//:local_config_sh.bzl", "sh_config_rule") + sh_config_rule(name = "local_config_sh") + ``` + +* **Bzlmod** + + With Bzlmod, you can introduce the same repository using a module extension, + which is similar to introducing the `@data_file` repository in the last + section. + + ``` + ## local_config_sh_extension.bzl + load("//:local_config_sh.bzl", "sh_config_rule") + + sh_config_extension = module_extension( + implementation = lambda ctx: sh_config_rule(name = "local_config_sh"), + ) + ``` + + Then use the extension in the MODULE.bazel file. + + ```python + ## MODULE.bazel + sh_config_ext = use_extension("//:local_config_sh_extension.bzl", "sh_config_extension") + use_repo(sh_config_ext, "local_config_sh") + ``` + +### Register toolchains & execution platforms {:#register-toolchains} + +Following the last section, after introducing a repository hosting toolchain +information (e.g. `local_config_sh`), you probably want to register the +toolchain. + +* **WORKSPACE** + + With WORKSPACE, you can register the toolchain in the following ways. + + 1. You can register the toolchain the `.bzl` file and load the macro in the + WORKSPACE file. + + ```python + ## local_config_sh.bzl + def sh_configure(): + sh_config_rule(name = "local_config_sh") + native.register_toolchains("@local_config_sh//:local_sh_toolchain") + ``` + + ```python + ## WORKSPACE + load("//:local_config_sh.bzl", "sh_configure") + sh_configure() + ``` + + 2. Or register the toolchain in the WORKSPACE file directly. + + ```python + ## WORKSPACE + load("//:local_config_sh.bzl", "sh_config_rule") + sh_config_rule(name = "local_config_sh") + register_toolchains("@local_config_sh//:local_sh_toolchain") + ``` + +* **Bzlmod** + + With Bzlmod, the + [`register_toolchains`](/rules/lib/globals/module#register_toolchains) and + [`register_execution_platforms`][register_execution_platforms] + APIs are only available in the MODULE.bazel file. You cannot call + `native.register_toolchains` in a module extension. + + ```python + ## MODULE.bazel + sh_config_ext = use_extension("//:local_config_sh_extension.bzl", "sh_config_extension") + use_repo(sh_config_ext, "local_config_sh") + register_toolchains("@local_config_sh//:local_sh_toolchain") + ``` + +[register_execution_platforms]: /rules/lib/globals/module#register_execution_platforms + +### Introduce local repositories {:#introduce-local-deps} + +You may need to introduce a dependency as a local repository when you need a +local version of the dependency for debugging or you want to incorporate a +directory in your workspace as external repository. + +* **WORKSPACE** + + With WORKSPACE, this is achieved by two native repository rules, + [`local_repository`](/reference/be/workspace#local_repository) and + [`new_local_repository`](/reference/be/workspace#new_local_repository). + + ```python + ## WORKSPACE + local_repository( + name = "rules_java", + path = "/Users/bazel_user/workspace/rules_java", + ) + ``` + +* **Bzlmod** + + With Bzlmod, you can use + [`local_path_override`](/rules/lib/globals/module#local_path_override) to + override a module with a local path. + + ```python + ## MODULE.bazel + bazel_dep(name = "rules_java") + local_path_override( + module_name = "rules_java", + path = "/Users/bazel_user/workspace/rules_java", + ) + ``` + + Note: With `local_path_override`, you can only introduce a local directory + as a Bazel module, which means it should have a MODULE.bazel file and its + transitive dependencies are taken into consideration during dependency + resolution. In addition, all module override directives can only be used by + the root module. + + It is also possible to introduce a local repository with module extension. + However, you cannot call `native.local_repository` in module extension, + there is ongoing effort on starlarkifying all native repository rules (check + [#18285](https://github.com/bazelbuild/bazel/issues/18285) for progress). + Then you can call the corresponding starlark `local_repository` in a module + extension. It's also trivial to implement a custom version of + `local_repository` repository rule if this is a blocking issue for you. + +### Bind targets {:#bind-targets} + +The [`bind`](/reference/be/workspace#bind) rule in WORKSPACE is deprecated and +not supported in Bzlmod. It was introduced to give a target an alias in the +special `//external` package. All users depending on this should migrate away. + +For example, if you have + +```python +## WORKSPACE +bind( + name = "openssl", + actual = "@my-ssl//src:openssl-lib", +) +``` + +This allows other targets to depend on `//external:openssl`. You can migrate +away from this by: + +* Replace all usages of `//external:openssl` with + `@my-ssl//src:openssl-lib`. + +* Or use the [`alias`](/reference/be/general#alias) build rule + * Define the following target in a package (e.g. `//third_party`) + + ```python + ## third_party/BUILD + alias( + name = "openssl, + actual = "@my-ssl//src:openssl-lib", + ) + ``` + + * Replace all usages of `//external:openssl` with + `//third_party:openssl-lib`. + +## Migration {:#migration} + +This section provides useful information and guidance for your Bzlmod migration +process. + +### Know your dependencies in WORKSPACE {:#know-deps-in-workspace} + +The first step of migration is to understand what dependencies you have. It +could be hard to figure out what exact dependencies are introduced in the +WORKSPACE file because transitive dependencies are often loaded with `*_deps` +macros. + +#### Inspect external dependency with workspace resolved file + +Fortunately, the flag +[`--experimental_repository_resolved_file`][resolved_file_flag] +can help. This flag essentially generates a "lock file" of all fetched external +dependencies in your last Bazel command. You can find more details in this [blog +post](https://blog.bazel.build/2018/07/09/bazel-sync-and-resolved-file.html). + +[resolved_file_flag]: /reference/command-line-reference#flag--experimental_repository_resolved_file + +It can be used in two ways: + +1. To fetch info of external dependencies needed for building certain targets. + + ```shell + bazel clean --expunge + bazel build --nobuild --experimental_repository_resolved_file=resolved.bzl //foo:bar + ``` + +2. To fetch info of all external dependencies defined in the WORKSPACE file. + + ```shell + bazel clean --expunge + bazel sync --experimental_repository_resolved_file=resolved.bzl + ``` + + With the `bazel sync` command, you can fetch all dependencies defined in the + WORKSPACE file, which include: + + * `bind` usages + * `register_toolchains` & `register_execution_platforms` usages + + However, if your project is cross platforms, bazel sync may break on certain + platforms because some repository rules may only run correctly on supported + platforms. + +After running the command, you should have information of your external +dependencies in the `resolved.bzl` file. + +#### Inspect external dependency with `bazel query` + +You may also know `bazel query` can be used for inspecting repository rules with + +```shell +bazel query --output=build //external: +``` + +While it is more convenient and much faster, [bazel query can lie about +external dependency version](https://github.com/bazelbuild/bazel/issues/12947), +so be careful using it! Querying and inspecting external +dependencies with Bzlmod is going to achieved by a [new +subcommand](https://github.com/bazelbuild/bazel/issues/15365). + +#### Built-in default dependencies {:#builtin-default-deps} + +If you check the file generated by `--experimental_repository_resolved_file`, +you are going to find many dependencies that are not defined in your WORKSPACE. +This is because Bazel in fact adds prefixes and suffixes to the user's WORKSPACE +file content to inject some default dependencies, which are usually required by +native rules (e.g. `@bazel_tools`, `@platforms` and `@remote_java_tools`). With +Bzlmod, those dependencies are introduced with a built-in module +[`bazel_tools`][bazel_tools] , which is a default dependency for every other +Bazel module. + +[bazel_tools]: https://github.com/bazelbuild/bazel/blob/master/src/MODULE.tools + +### Hybrid mode for gradual migration {:#hybrid-mode} + +Bzlmod and WORKSPACE can work side by side, which allows migrating dependencies +from the WORKSPACE file to Bzlmod to be a gradual process. + +#### WORKSPACE.bzlmod {:#workspace.bzlmod} + +During the migration, Bazel users may need to switch between builds with and +without Bzlmod enabled. WORKSPACE.bzlmod support is implemented to make the +process smoother. + +WORKSPACE.bzlmod has the exact same syntax as WORKSPACE. When Bzlmod is enabled, +if a WORKSPACE.bzlmod file also exists at the workspace root: + +* `WORKSPACE.bzlmod` takes effect and the content of `WORKSPACE` is ignored. +* No [prefixes or suffixes](/external/migration#builtin-default-deps) are + added to the WORKSPACE.bzlmod file. + +Using the WORKSPACE.bzlmod file can make the migration easier because: + +* When Bzlmod is disabled, you fall back to fetching dependencies from the + original WORKSPACE file. +* When Bzlmod is enabled, you can better track what dependencies are left to + migrate with WORKSPACE.bzlmod. + +Note: WORKSPACE.bzlmod does NOT replace the functionality of WORKSPACE for +identifying the workspace root, therefore you still need a WORKSPACE file at +your workspace root. + +#### Repository visibility {:#repository-visibility} + +Bzlmod is able to control which other repositories are visible from a given +repository, check [repository names and strict +deps](/external/module#repository_names_and_strict_deps) for more details. + +Here is a summary of repository visibilities from different types of +repositories when also taking WORKSPACE into consideration. + +| | From the main repo | From Bazel module repos | From module extension repos | From WORKSPACE repos | +|----------------|--------------------|-------------------------|---------------------------------------------------------------------------------------------------------------------|----------------------| +| The main repo | Visible | If the root module is a direct dependency | If the root module is a direct dependency of the module hosting the module extension | Visible | +| Bazel module repos | Direct deps | Direct deps | Direct deps of the module hosting the module extension | Direct deps of the root module | +| Module extension repos | Direct deps | Direct deps | Direct deps of the module hosting the module extension + all repos generated by the same module extension | Direct deps of the root module | +| WORKSPACE Repos | All visible | Not visible | Not visible | All visible | + +Note: For the root module, if a repository `@foo` is defined in WORKSPACE and +`@foo` is also used as an [apparent repository +name](/external/overview#apparent-repo-name) in MODULE.bazel, then `@foo` +refers to the one introduced in MODULE.bazel. + +Note: For a module extension generated repository `@bar`, if `@foo` is used as +an [apparent repository name](/external/overview#apparent-repo-name) of +another repository generated by the same module extension and direct +dependencies of the module hosting the module extension, then for repository +`@bar`, `@foo` refers to the latter. + +### Migration process {:#migration-process} + +A typical Bzlmod migration process can look like this: + +1. Understand what dependencies you have in WORKSPACE. +1. Add an empty MODULE.bazel file at your project root. +1. Add an empty WORKSPACE.bzlmod file to override the WORKSPACE file content. +1. Build your targets with Bzlmod enabled and check which repository is + missing. +1. Check the definition of the missing repository in the resolved dependency + file. +1. Introduce the missing dependency as a Bazel module, through a module + extension, or leave it in the WORKSPACE.bzlmod for later migration. +1. Go back to 4 and repeat until all dependencies are available. + +#### Migration tool {:#migration-tool} + +There is an interactive Bzlmod migration [helper script][migration_script] that +can get you started. + +[migration_script]: https://github.com/bazelbuild/bazel-central-registry/blob/main/tools/migrate_to_bzlmod.py + +The script does the following things: + +* Generate and parse the WORKSPACE resolved file. +* Print repository info from the resolved file in a human readable way. +* Run bazel build command, detect recognized error messages, and recommend a + way to migrate. +* Check if a dependency is already available in the BCR. +* Add a dependency to MODULE.bazel file. +* Add a dependency through a module extension. +* Add a dependency to WORKSPACE.bzlmod file. + +To use it, make sure you have the latest Bazel release installed, and run the +following command: + +```shell +git clone https://github.com/bazelbuild/bazel-central-registry.git +cd +/tools/migrate_to_bzlmod.py -t +``` + +Note: The migration script is not perfect and may not be up-to-date since Bzlmod +is evolving, always double check if the recommended solution is correct. + +## Publish Bazel modules {:#publish-modules} + +If your Bazel project is a dependency for other projects, you can publish your +project in the [Bazel Central Registry](https://registry.bazel.build/). + +To be able to check in your project in the BCR, you need a source archive URL of +the project. Take note of a few things when creating the source archive: + +* **Make sure the archive is pointing to a specific version.** + + The BCR can only accept versioned source archives because Bzlmod needs to + conduct version comparison during dependency resolution. + +* **Make sure the archive URL is stable.** + + Bazel verifies the content of the archive by a hash value, so you should + make sure the checksum of the downloaded file never changes. If the URL is + from GitHub, please create and upload a release archive in the release page. + GitHub isn't going to guarantee the checksum of source archives generated on + demand. In short, URLs in the form of + `https://github.com///releases/download/...` is considered stable + while `https://github.com///archive/...` is not. Check [GitHub + Archive Checksum + Outage](https://blog.bazel.build/2023/02/15/github-archive-checksum.html) + for more context. + +* **Make sure the source tree follows the layout of the original repository.** + + In case your repository is very large and you want to create a distribution + archive with reduced size by stripping out unnecessary sources, please make + sure the stripped source tree is a subset of the original source tree. This + makes it easier for end users to override the module to a non-release + version by [`archive_override`](/rules/lib/globals/module#archive_override) + and [`git_override`](/rules/lib/globals/module#git_override). + +* **Include a test module in a subdirectory that tests your most common + APIs.** + + A test module is a Bazel project with its own WORKSPACE and MODULE.bazel + file located in a subdirectory of the source archive which depends on the + actual module to be published. It should contain examples or some + integration tests that cover your most common APIs. Check + [test module][test_module] to learn how to set it up. + +[test_module]: https://github.com/bazelbuild/bazel-central-registry/tree/main/docs#test-module + +When you have your source archive URL ready, follow the [BCR contribution +guidelines][bcr_contrib_guide] to submit your module to the BCR with a GitHub +Pull Request. + +[bcr_contrib_guide]: https://github.com/bazelbuild/bazel-central-registry/tree/main/docs#contribute-a-bazel-module + +It is **highly recommended** to set up the [Publish to +BCR](https://github.com/bazel-contrib/publish-to-bcr) GitHub App for your +repository to automate the process of submitting your module to the BCR. + +## Best practices {:#best-practices} + +This section documents a few best practices you should follow for better +managing your external dependencies. + +#### Split targets into different packages to avoid fetching unnecessary dependencies. + +Check [#12835](https://github.com/bazelbuild/bazel/issues/12835), where dev +dependencies for tests are forced to be fetched unnecessarily for building +targets that don't need them. This is not actually not Bzlmod specific, but +following this practices makes it easier to specify dev dependencies correctly. + +#### Specify dev dependencies + +You can set the `dev_dependency` attribute to true for +[`bazel_dep`](/rules/lib/globals/module#bazel_dep) and +[`use_extension`](/rules/lib/globals/module#use_extension) directives so that +they don't propagate to dependent projects. As the root module, you can use the +[`--ignore_dev_dependency`][ignore_dev_dep_flag] flag to verify if your targets +still build without dev dependencies. + +[ignore_dev_dep_flag]: /reference/command-line-reference#flag--ignore_dev_dependency + +{# More best practices here !!! #} + +## Community migration progress {:#migration-progress} + +You can check the [Bazel Central Registry](https://registry.bazel.build) to find +out if your dependencies are already available. Otherwise feel free to join this +[GitHub discussion](https://github.com/bazelbuild/bazel/discussions/18329) to +upvote or post the dependencies that are blocking your migration. + +## Report issues {:#reporting-issues} + +Please check the [Bazel GitHub issue list][bzlmod_github_issue] for known Bzlmod +issues. Feel free to file new issues or feature requests that can help unblock +your migration! + +[bzlmod_github_issue]: https://github.com/bazelbuild/bazel/issues?q=is%3Aopen+is%3Aissue+label%3Aarea-Bzlmod diff --git a/site/en/external/mod-command.md b/site/en/external/mod-command.md new file mode 100644 index 00000000000000..dd425c1ecc6434 --- /dev/null +++ b/site/en/external/mod-command.md @@ -0,0 +1,547 @@ +Project: /_project.yaml +Book: /_book.yaml +keywords: Bzlmod + +# `mod` Command + +{% include "_buttons.html" %} + +The `mod` command, introduced in Bazel 6.3.0, provides a range of tools to help +the user understand their external dependency graph when Bzlmod is enabled. It +lets you visualize the dependency graph, find out why a certain module or a +version of a module is present in the graph, view the repo definitions backing +modules, inspect usages of module extensions and repos they generate, among +other functions. + +## Syntax {:#syntax} + +```sh +bazel mod [] [ [...]] +``` + +The available subcommands and their respective required arguments are: + +* `graph`: Displays the full dependency graph of the project, starting from + the root module. If one or more modules are specified in `--from`, these + modules are shown directly under the root, and the graph is only expanded + starting from them (see [example](#mod-example1)). + +* `deps ...`: Displays the resolved direct dependencies of each of the + specified modules, similarly to `graph`. + +* `all_paths ...`: Displays all existing paths from the root to the + specified `...`. If one or more modules are specified in `--from`, + these modules are shown directly under the root, and the graph contains any + any existing path from the `--from` modules to the argument modules (see + [example](#mod-example4)). + +* `path ...`: Has the same semantics as `all_paths`, but only display a + single path from one of the `--from` modules to one of the argument modules. + +* `explain ...`: Shows all the places where the specified modules appear + in the dependency graph, along with the modules that directly depend on + them. The output of the `explain` command is essentially a pruned version of + the `all_paths` command, containing 1) the root module; 2) the root module's + direct dependencies that lead to the argument modules; 3) the argument + modules' direct dependents; and + 4) the argument modules themselves (see [example](#mod-example5)). + +* `show_repo ...`: Displays the definition of the specified repos (see + [example](#mod-example6)). + +* `show_extension ...`: Displays information about each of the + specified extensions: a list of the generated repos along with the modules + that import them using `use_repo`, and a list of the usages of that + extension in each of the modules where it is used, containing the specified + tags and the `use_repo` calls (see [example](#mod-example8)). + +`` refers to one or more modules or repos. It can be one of: + +* The literal string ``: The root module representing your current + project. + +* `@`: The module `` at version ``. For a module + with a non-registry override, use an underscore (`_`) as the ``. + +* ``: All present versions of the module ``. + +* `@`: The repo with the given [apparent + name](overview#apparent-repo-name) in the context of the `--base_module`. + +* `@@`: The repo with the given [canonical + name](overview#canonical-repo-name). + +In a context requiring specifying modules, ``s referring to repos that +correspond to modules (as opposed to extension-generated repos) can also be +used. Conversely, in a context requiring specifying repos, ``s referring to +modules can stand in for the corresponding repos. + +`` must be of the form `%`. +The `` part must be a repo-relative label (for example, +`//pkg/path:file.bzl`). + +The following options only affect the subcommands that print graphs (`graph`, +`deps`, `all_paths`, `path`, and `explain`): + +* `--from [,[,...]]` *default: ``*: The module(s) from which + the graph is expanded in `graph`, `all_paths`, `path`, and `explain`. Check + the subcommands' descriptions for more details. + +* `--verbose` *default: "false"*: Include in the output graph extra + information about the version resolution of each module. If the module + version changed during resolution, show either which version replaced it or + what was the original version, the reason it was replaced, and which modules + requested the new version if the reason was [Minimal Version + Selection](module#version-selection). + +* `--include_unused` *default: "false"*: Include in the output graph the + modules which were originally present in the dependency graph, but became + unused after module resolution. + +* `--extension_info `: Include information about the module extension + usages as part of the output graph (see [example](#mod-example7)). `` + can be one of: + + * `hidden` *(default)*: Don't show anything about extensions. + + * `usages`: Show extensions under each module where they are used. They + are printed in the form of `$`. + + * `repos`: In addition to `usages`, show the repo imported using + `use_repo` under each extension usage. + + * `all`: In addition to `usages` and `repos`, also show + extension-generated repos that are not imported by any module. These + extra repos are shown under the first occurrence of their generating + extension in the output, and are connected with a dotted edge. + +* `--extension_filter [,[,...]]`: If specified, the + output graph only includes modules that use the specified extensions, and + the paths that lead to those modules. Specifying an empty extension list (as + in `--extension_filter=`) is equivalent to specifying _all_ extensions used + by any module in the dependency graph. + +* `--depth `: The depth of the output graph. A depth of 1 only displays the + root and its direct dependencies. Defaults to 1 for `explain`, 2 for `deps` + and infinity for the others. + +* `--cycles` *default: "false"*: Include cycle edges in the output graph. + +* `--include_builtin` *default: "false"*: Include built-in modules (such as + `@bazel_tools`) in the output graph. This flag is disabled by default, as + built-in modules are implicitly depended on by every other module, which + greatly clutters the output. + +* `--charset ` *default: utf8*: Specify the charset to use for text + output. Valid values are `"utf8"` and `"ascii"`. The only significant + difference is in the special characters used to draw the graph in the + `"text"` output format, which don't exist in the `"ascii"` charset. + Therefore, the `"ascii"` charset is present to also support the usage on + legacy platforms which cannot use Unicode. + +* `--output `: Include information about the module extension usages as + part of the output graph. ` can be one of: + + * `text` *(default)*: A human-readable representation of the output graph + (flattened as a tree). + + * `json`: Outputs the graph in the form of a JSON object (flattened as a + tree). + + * `graph`: Outputs the graph in the Graphviz *dot* representation. + + Tip: Use the following command to pipe the output through the *dot* engine + and export the graph representation as an SVG image. + + ```sh + bazel mod graph --output graph | dot -Tsvg > /tmp/graph.svg + ``` + +Other options include: + +* `--base_module ` *default: ``*: Specify a module relative to + which apparent repo names in arguments are interpreted. Note that this + argument itself can be in the form of `@`; this is always + interpreted relative to the root module. + +* `--extension_usages [,[,...]]`: Filters `show_extension` to only + display extension usages from the specified modules. + +## Examples {:#examples} + +Some possible usages of the `mod` command on a real Bazel project are showcased +below to give you a general idea on how you can use it to inspect your project's +external dependencies. + +`MODULE.bazel` file: + +```python +module( + name = "my_project", + version = "1.0", +) + +bazel_dep(name = "bazel_skylib", version = "1.1.1", repo_name = "skylib1") +bazel_dep(name = "bazel_skylib", version = "1.2.0", repo_name = "skylib2") +multiple_version_override(module_name = "bazel_skylib", versions = ["1.1.1", "1.2.0"]) + +bazel_dep(name = "stardoc", version = "0.5.0") +bazel_dep(name = "rules_java", version = "5.0.0") + +toolchains = use_extension("@rules_java//java:extensions.bzl", "toolchains") +use_repo(toolchains, my_jdk="remotejdk17_linux") +``` + + + + + + +
+
+ Graph Before Resolution +
Graph Before Resolution
+
+ + 		+
+ Graph After Resolution +
Graph After Resolution
+
+ +
+ +1. Display the whole dependency graph of your + project. + + ```sh + bazel mod graph + ``` + + ```none + (my_project@1.0) + ├───bazel_skylib@1.1.1 + │ └───platforms@0.0.4 + ├───bazel_skylib@1.2.0 + │ └───platforms@0.0.4 ... + ├───rules_java@5.0.0 + │ ├───platforms@0.0.4 ... + │ ├───rules_cc@0.0.1 + │ │ ├───bazel_skylib@1.1.1 ... + │ │ └───platforms@0.0.4 ... + │ └───rules_proto@4.0.0 + │ ├───bazel_skylib@1.1.1 ... + │ └───rules_cc@0.0.1 ... + └───stardoc@0.5.0 + ├───bazel_skylib@1.1.1 ... + └───rules_java@5.0.0 ... + ``` + + Note: The `...` symbol indicates that the node has already been expanded + somewhere else and was not expanded again to reduce noise. + +2. Display the whole dependency graph (including + unused modules and with extra information about version resolution). + + ```sh + bazel mod graph --include_unused --verbose + ``` + + ```none + (my_project@1.0) + ├───bazel_skylib@1.1.1 + │ └───platforms@0.0.4 + ├───bazel_skylib@1.2.0 + │ └───platforms@0.0.4 ... + ├───rules_java@5.0.0 + │ ├───platforms@0.0.4 ... + │ ├───rules_cc@0.0.1 + │ │ ├───bazel_skylib@1.0.3 ... (to 1.1.1, cause multiple_version_override) + │ │ ├───bazel_skylib@1.1.1 ... (was 1.0.3, cause multiple_version_override) + │ │ └───platforms@0.0.4 ... + │ └───rules_proto@4.0.0 + │ ├───bazel_skylib@1.0.3 ... (to 1.1.1, cause multiple_version_override) + │ ├───bazel_skylib@1.1.1 ... (was 1.0.3, cause multiple_version_override) + │ └───rules_cc@0.0.1 ... + └───stardoc@0.5.0 + ├───bazel_skylib@1.1.1 ... (was 1.0.3, cause multiple_version_override) + ├───rules_java@5.0.0 ... (was 4.0.0, cause , bazel_tools@_) + ├───bazel_skylib@1.0.3 (to 1.1.1, cause multiple_version_override) + │ └───platforms@0.0.4 ... + └───rules_java@4.0.0 (to 5.0.0, cause , bazel_tools@_) + ├───bazel_skylib@1.0.3 ... (to 1.1.1, cause multiple_version_override) + └───bazel_skylib@1.1.1 ... (was 1.0.3, cause multiple_version_override) + ``` + +3. Display the dependency graph expanded from + some specific modules. + + ```sh + bazel mod graph --from rules_java --include_unused + ``` + + ```none + (my_project@1.0) + ├───rules_java@5.0.0 + │ ├───platforms@0.0.4 + │ ├───rules_cc@0.0.1 + │ │ ├───bazel_skylib@1.0.3 ... (unused) + │ │ ├───bazel_skylib@1.1.1 ... + │ │ └───platforms@0.0.4 ... + │ └───rules_proto@4.0.0 + │ ├───bazel_skylib@1.0.3 ... (unused) + │ ├───bazel_skylib@1.1.1 ... + │ └───rules_cc@0.0.1 ... + └╌╌rules_java@4.0.0 (unused) + ├───bazel_skylib@1.0.3 (unused) + │ └───platforms@0.0.4 ... + └───bazel_skylib@1.1.1 + └───platforms@0.0.4 ... + ``` + + Note: The dotted line is used to indicate an *indirect* (transitive) + dependency edge between two nodes. + +4. Display all paths between two of your + modules. + + ```sh + bazel mod all_paths bazel_skylib@1.1.1 --from rules_proto + ``` + + ```none + (my_project@1.0) + └╌╌rules_proto@4.0.0 + ├───bazel_skylib@1.1.1 + └───rules_cc@0.0.1 + └───bazel_skylib@1.1.1 ... + ``` + +5. See why and how your project depends on some + module(s). + + ```sh + bazel mod explain @skylib1 --verbose --include_unused + ``` + + ```none + (my_project@1.0) + ├───bazel_skylib@1.1.1 + ├───rules_java@5.0.0 + │ ├───rules_cc@0.0.1 + │ │ └───bazel_skylib@1.1.1 ... (was 1.0.3, cause multiple_version_override) + │ └───rules_proto@4.0.0 + │ ├───bazel_skylib@1.1.1 ... (was 1.0.3, cause multiple_version_override) + │ └───rules_cc@0.0.1 ... + └───stardoc@0.5.0 + ├───bazel_skylib@1.1.1 ... (was 1.0.3, cause multiple_version_override) + ├╌╌rules_cc@0.0.1 + │ └───bazel_skylib@1.1.1 ... (was 1.0.3, cause multiple_version_override) + └╌╌rules_proto@4.0.0 + ├───bazel_skylib@1.1.1 ... (was 1.0.3, cause multiple_version_override) + └───rules_cc@0.0.1 ... + ``` + +6. See the underlying rule of some your modules' + repos. + + ```sh + bazel mod show_repo rules_cc stardoc + ``` + + ```none + ## rules_cc@0.0.1: + # + http_archive( + name = "rules_cc~0.0.1", + urls = ["https://bcr.bazel.build/test-mirror/github.com/bazelbuild/rules_cc/releases/download/0.0.1/rules_cc-0.0.1.tar.gz", "https://github.com/bazelbuild/rules_cc/releases/download/0.0.1/rules_cc-0.0.1.tar.gz"], + integrity = "sha256-Tcy/0iwN7xZMj0dFi9UODHFI89kgAs20WcKpamhJgkE=", + strip_prefix = "", + remote_patches = {"https://bcr.bazel.build/modules/rules_cc/0.0.1/patches/add_module_extension.patch": "sha256-g3+zmGs0YT2HKOVevZpN0Jet89Ylw90Cp9XsIAY8QqU="}, + remote_patch_strip = 1, + ) + # Rule http_archive defined at (most recent call last): + # /home/user/.cache/bazel/_bazel_user/6e893e0f5a92cc4cf5909a6e4b2770f9/external/bazel_tools/tools/build_defs/repo/http.bzl:355:31 in + + ## stardoc: + # + http_archive( + name = "stardoc~0.5.0", + urls = ["https://bcr.bazel.build/test-mirror/github.com/bazelbuild/stardoc/releases/download/0.5.0/stardoc-0.5.0.tar.gz", "https://github.com/bazelbuild/stardoc/releases/download/0.5.0/stardoc-0.5.0.tar.gz"], + integrity = "sha256-yXlNzIAmow/2fPfPkeviRcopSyCwcYRdEsGSr+JDrXI=", + strip_prefix = "", + remote_patches = {}, + remote_patch_strip = 0, + ) + # Rule http_archive defined at (most recent call last): + # /home/user/.cache/bazel/_bazel_user/6e893e0f5a92cc4cf5909a6e4b2770f9/external/bazel_tools/tools/build_defs/repo/http.bzl:355:31 in + ``` + +7. See what module extensions are used in your + dependency graph. + + ```sh + bazel mod graph --extension_info=usages --extension_filter=all + ``` + + ```none + (my_project@1.0) + ├───$@@rules_java.5.0.0//java:extensions.bzl%toolchains + ├───rules_java@5.0.0 # + │ ├───$@@rules_java.5.0.0//java:extensions.bzl%toolchains + │ ├───rules_cc@0.0.1 # + │ │ └───$@@rules_cc.0.0.1//bzlmod:extensions.bzl%cc_configure + │ └───rules_proto@4.0.0 + │ └───rules_cc@0.0.1 ... + └───stardoc@0.5.0 + └───rules_java@5.0.0 ... + ``` + +8. See what repositories are generated and + imported from some specific extension as part of the dependency graph. + + ```sh + bazel mod show_extension @@rules_java~5.0.0//java:extensions.bzl%toolchains + ``` + + ```none + (my_project@1.0) + ├───$@@rules_java.5.0.0//java:extensions.bzl%toolchains + │ ├───remotejdk17_linux + │ ├╌╌remotejdk11_linux + │ ├╌╌remotejdk11_linux_aarch64 + │ ├╌╌remotejdk11_linux_ppc64le + │ ├╌╌remotejdk11_linux_s390x + ...(some lines omitted)... + ├───rules_java@5.0.0 # + │ └───$@@rules_java.5.0.0//java:extensions.bzl%toolchains ... + │ ├───local_jdk + │ ├───remote_java_tools + │ ├───remote_java_tools_darwin + │ ├───remote_java_tools_linux + │ ├───remote_java_tools_windows + │ ├───remotejdk11_linux_aarch64_toolchain_config_repo + │ ├───remotejdk11_linux_ppc64le_toolchain_config_repo + ...(some lines omitted)... + └───stardoc@0.5.0 + └───rules_java@5.0.0 ... + ``` + +9. See the list of generated repositories of an + extension and how that extension is used in each module. + + ```sh + bazel mod graph --extension_info=all --extension_filter=@rules_java//java:extensions.bzl%toolchains + ``` + + ```none + ## @@rules_java.5.0.0//java:extensions.bzl%toolchains: + + Fetched repositories: + - local_jdk (imported by bazel_tools@_, rules_java@5.0.0) + - remote_java_tools (imported by bazel_tools@_, rules_java@5.0.0) + - remote_java_tools_darwin (imported by bazel_tools@_, rules_java@5.0.0) + - remote_java_tools_linux (imported by bazel_tools@_, rules_java@5.0.0) + - remote_java_tools_windows (imported by bazel_tools@_, rules_java@5.0.0) + - remotejdk11_linux_aarch64_toolchain_config_repo (imported by rules_java@5.0.0) + - remotejdk11_linux_ppc64le_toolchain_config_repo (imported by rules_java@5.0.0) + ...(some lines omitted)... + - remotejdk17_linux (imported by ) + - remotejdk11_linux + - remotejdk11_linux_aarch64 + - remotejdk11_linux_ppc64le + - remotejdk11_linux_s390x + - remotejdk11_macos + ...(some lines omitted)... + + # Usage in at /MODULE.bazel:14:27 with the specified attributes: + use_repo( + toolchains, + my_jdk="remotejdk17_linux", + ) + + # Usage in bazel_tools@_ at bazel_tools@_/MODULE.bazel:23:32 with the specified attributes: + use_repo( + toolchains, + "local_jdk", + "remote_java_tools", + "remote_java_tools_linux", + "remote_java_tools_windows", + "remote_java_tools_darwin", + ) + + # Usage in rules_java@5.0.0 at rules_java@5.0.0/MODULE.bazel:30:27 with the specified attributes: + use_repo( + toolchains, + "remote_java_tools", + "remote_java_tools_linux", + "remote_java_tools_windows", + "remote_java_tools_darwin", + "local_jdk", + "remotejdk11_linux_toolchain_config_repo", + "remotejdk11_macos_toolchain_config_repo", + "remotejdk11_macos_aarch64_toolchain_config_repo", + ...(some lines omitted)... + ) + ``` + +10. See the underlying rule of some + extension-generated repositories. + + ```sh + bazel mod show_repo --base_module=rules_java @remote_java_tools + ``` + + ```none + ## @remote_java_tools: + # + http_archive( + name = "rules_java~5.0.0~toolchains~remote_java_tools", + urls = ["https://mirror.bazel.build/bazel_java_tools/releases/java/v11.5/java_tools-v11.5.zip", "https://github.com/bazelbuild/java_tools/releases/download/java_v11.5/java_tools-v11.5.zip"], + sha256 = "b763ee80e5754e593fd6d5be6d7343f905bc8b73d661d36d842b024ca11b6793", + ) + # Rule http_archive defined at (most recent call last): + # /home/user/.cache/bazel/_bazel_user/6e893e0f5a92cc4cf5909a6e4b2770f9/external/bazel_tools/tools/build_defs/repo/http.bzl:355:31 in + ``` \ No newline at end of file diff --git a/site/en/external/module.md b/site/en/external/module.md new file mode 100644 index 00000000000000..8e010e96fab3cf --- /dev/null +++ b/site/en/external/module.md @@ -0,0 +1,193 @@ +Project: /_project.yaml +Book: /_book.yaml + +# Bazel modules + +{% include "_buttons.html" %} + +A Bazel **module** is a Bazel project that can have multiple versions, each of +which publishes metadata about other modules that it depends on. This is +analogous to familiar concepts in other dependency management systems, such as a +Maven *artifact*, an npm *package*, a Go *module*, or a Cargo *crate*. + +A module must have a `MODULE.bazel` file at its repo root (next to the +`WORKSPACE` file). This file is the module's manifest, declaring its name, +version, list of direct dependencies, and other information. For a basic +example: + +```python +module(name = "my-module", version = "1.0") + +bazel_dep(name = "rules_cc", version = "0.0.1") +bazel_dep(name = "protobuf", version = "3.19.0") +``` + + + +To perform module resolution, Bazel starts by reading the root module's +`MODULE.bazel` file, and then repeatedly requests any dependency's +`MODULE.bazel` file from a [Bazel registry](/external/registry) until it +discovers the entire dependency graph. + +By default, Bazel then [selects](#version-selection) one version of each module +to use. Bazel represents each module with a repo, and consults the registry +again to learn how to define each of the repos. + +## Version format + +Bazel has a diverse ecosystem and projects use various versioning schemes. The +most popular by far is [SemVer](https://semver.org){: .external}, but there are +also prominent projects using different schemes such as +[Abseil](https://github.com/abseil/abseil-cpp/releases){: .external}, whose +versions are date-based, for example `20210324.2`). + +For this reason, Bzlmod adopts a more relaxed version of the SemVer spec. The +differences include: + +* SemVer prescribes that the "release" part of the version must consist of 3 + segments: `MAJOR.MINOR.PATCH`. In Bazel, this requirement is loosened so + that any number of segments is allowed. +* In SemVer, each of the segments in the "release" part must be digits only. + In Bazel, this is loosened to allow letters too, and the comparison + semantics match the "identifiers" in the "prerelease" part. +* Additionally, the semantics of major, minor, and patch version increases are + not enforced. However, see [compatibility level](#compatibility_level) for + details on how we denote backwards compatibility. + +Any valid SemVer version is a valid Bazel module version. Additionally, two +SemVer versions `a` and `b` compare `a < b` if and only if the same holds when +they're compared as Bazel module versions. + +## Version selection {:#version-selection} + +Consider the diamond dependency problem, a staple in the versioned dependency +management space. Suppose you have the dependency graph: + +``` + A 1.0 + / \ + B 1.0 C 1.1 + | | + D 1.0 D 1.1 +``` + +Which version of `D` should be used? To resolve this question, Bzlmod uses the +[Minimal Version Selection](https://research.swtch.com/vgo-mvs){: .external} +(MVS) algorithm introduced in the Go module system. MVS assumes that all new +versions of a module are backwards compatible, and so picks the highest version +specified by any dependent (`D 1.1` in our example). It's called "minimal" +because `D 1.1` is the earliest version that could satisfy our requirements — +even if `D 1.2` or newer exists, we don't select them. Using MVS creates a +version selection process that is *high-fidelity* and *reproducible*. + +### Yanked versions + +The registry can declare certain versions as *yanked* if they should be avoided +(such as for security vulnerabilities). Bazel throws an error when selecting a +yanked version of a module. To fix this error, either upgrade to a newer, +non-yanked version, or use the +[`--allow_yanked_versions`](/reference/command-line-reference#flag--allow_yanked_versions) +flag to explicitly allow the yanked version. + +## Compatibility level + +In Go, MVS's assumption about backwards compatibility works because it treats +backwards incompatible versions of a module as a separate module. In terms of +SemVer, that means `A 1.x` and `A 2.x` are considered distinct modules, and can +coexist in the resolved dependency graph. This is, in turn, made possible by +encoding the major version in the package path in Go, so there aren't any +compile-time or linking-time conflicts. + +Bazel, however, cannot provide such guarantees, so it needs the "major version" +number in order to detect backwards incompatible versions. This number is called +the *compatibility level*, and is specified by each module version in its +`module()` directive. With this information, Bazel can throw an error when it +detects that versions of the same module with different compatibility levels +exist in the resolved dependency graph. + +## Overrides + +Specify overrides in the `MODULE.bazel` file to alter the behavior of Bazel +module resolution. Only the root module's overrides take effect — if a module is +used as a dependency, its overrides are ignored. + +Each override is specified for a certain module name, affecting all of its +versions in the dependency graph. Although only the root module's overrides take +effect, they can be for transitive dependencies that the root module does not +directly depend on. + +### Single-version override + +The [`single_version_override`](/rules/lib/globals#single_version_override) +serves multiple purposes: + +* With the `version` attribute, you can pin a dependency to a specific + version, regardless of which versions of the dependency are requested in the + dependency graph. +* With the `registry` attribute, you can force this dependency to come from a + specific registry, instead of following the normal [registry + selection](/external/registry#selecting_registries) process. +* With the `patch*` attributes, you can specify a set of patches to apply to + the downloaded module. + +These attributes are all optional and can be mixed and matched with each other. + +### Multiple-version override + +A [`multiple_version_override`](/rules/lib/globals#multiple_version_override) +can be specified to allow multiple versions of the same module to coexist in the +resolved dependency graph. + +You can specify an explicit list of allowed versions for the module, which must +all be present in the dependency graph before resolution — there must exist +*some* transitive dependency depending on each allowed version. After +resolution, only the allowed versions of the module remain, while Bazel upgrades +other versions of the module to the nearest higher allowed version at the same +compatibility level. If no higher allowed version at the same compatibility +level exists, Bazel throws an error. + +For example, if versions `1.1`, `1.3`, `1.5`, `1.7`, and `2.0` exist in the +dependency graph before resolution and the major version is the compatibility +level: + +* A multiple-version override allowing `1.3`, `1.7`, and `2.0` results in + `1.1` being upgraded to `1.3`, `1.5` being upgraded to `1.7`, and other + versions remaining the same. +* A multiple-version override allowing `1.5` and `2.0` results in an error, as + `1.7` has no higher version at the same compatibility level to upgrade to. +* A multiple-version override allowing `1.9` and `2.0` results in an error, as + `1.9` is not present in the dependency graph before resolution. + +Additionally, users can also override the registry using the `registry` +attribute, similarly to single-version overrides. + +### Non-registry overrides + +Non-registry overrides completely remove a module from version resolution. Bazel +does not request these `MODULE.bazel` files from a registry, but instead from +the repo itself. + +Bazel supports the following non-registry overrides: + +* [`archive_override`](/rules/lib/globals#archive_override) +* [`git_override`](/rules/lib/globals#git_override) +* [`local_path_override`](/rules/lib/globals#local_path_override) + +## Repository names and strict deps + +The [canonical name](/external/overview#canonical-repo-name) of a repo backing a +module is `{{ "" }}module_name{{ "" }}~{{ "" }}version{{ +"" }}` (for example, `bazel_skylib~1.0.3`). For modules with a +non-registry override, replace the `{{ "" }}version{{ "" }}` part +with the string `override`. Note that the canonical name format is not an API +you should depend on and is subject to change at any time. + +The [apparent name](/external/overview#apparent-repo-name) of a repo backing a +module to its direct dependents defaults to its module name, unless the +`repo_name` attribute of the [`bazel_dep`](/rules/lib/globals#bazel_dep) +directive says otherwise. Note that this means a module can only find its direct +dependencies. This helps prevent accidental breakages due to changes in +transitive dependencies. + +[Module extensions](/external/extension) can also introduce additional repos +into the visible scope of a module. diff --git a/site/en/external/overview.md b/site/en/external/overview.md new file mode 100644 index 00000000000000..f7162354a5e245 --- /dev/null +++ b/site/en/external/overview.md @@ -0,0 +1,196 @@ +Project: /_project.yaml +Book: /_book.yaml + +# External dependencies overview + +{% include "_buttons.html" %} + +Bazel supports *external dependencies*, source files (both text and binary) used +in your build that are not from your workspace. For example, they could be a +ruleset hosted in a GitHub repo, a Maven artifact, or a directory on your local +machine outside your current workspace. + +As of Bazel 6.0, there are two ways to manage external dependencies with Bazel: +the traditional, repository-focused [`WORKSPACE` system](#workspace-system), and +the newer module-focused [`MODULE.bazel` system](#bzlmod) (codenamed *Bzlmod*, +and enabled with the flag `--enable_bzlmod`). The two systems can be used +together, but Bzlmod is replacing the `WORKSPACE` system in future Bazel +releases, check the [Bzlmod migration guide](/external/migration) on how to +migrate. + +This document explains the concepts surrounding external dependency management +in Bazel, before going into a bit more detail about the two systems in order. + +## Concepts {:#concepts} + +### Repository {:#repository} + +A directory with a `WORKSPACE` or `WORKSPACE.bazel` file, containing source +files to be used in a Bazel build. Often shortened to just **repo**. + +### Main repository {:#main-repository} + +The repository in which the current Bazel command is being run. + +### Workspace {:#workspace} + +The environment shared by all Bazel commands run in the same main repository. + +Note that historically the concepts of "repository" and "workspace" have been +conflated; the term "workspace" has often been used to refer to the main +repository, and sometimes even used as a synonym of "repository". + +### Canonical repository name {:#canonical-repo-name} + +The canonical name a repository is addressable by. Within the context of a +workspace, each repository has a single canonical name. A target inside a repo +whose canonical name is `canonical_name` can be addressed by the label +`@@canonical_name//pac/kage:target` (note the double `@`). + +The main repository always has the empty string as the canonical name. + +### Apparent repository name {:#apparent-repo-name} + +The name a repository is addressable by in the context of a certain other repo. +This can be thought of as a repo's "nickname": The repo with the canonical name +`michael` might have the apparent name `mike` in the context of the repo +`alice`, but might have the apparent name `mickey` in the context of the repo +`bob`. In this case, a target inside `michael` can be addressed by the label +`@mike//pac/kage:target` in the context of `alice` (note the single `@`). + +Conversely, this can be understood as a **repository mapping**: each repo +maintains a mapping from "apparent repo name" to a "canonical repo name". + +### Repository rule {:#repo-rule} + +A schema for repository definitions that tells Bazel how to materialize a +repository. For example, it could be "download a zip archive from a certain URL +and extract it", or "fetch a certain Maven artifact and make it available as a +`java_import` target", or simply "symlink a local directory". Every repo is +**defined** by calling a repo rule with an appropriate number of arguments. + +See [Repository rules](/extending/repo) for more information about how to write +your own repository rules. + +The most common repo rules by far are +[`http_archive`](/rules/lib/repo/http#http_archive), which downloads an archive +from a URL and extracts it, and +[`local_repository`](/reference/be/workspace#local_repository), which symlinks a +local directory that is already a Bazel repository. + +### Fetch a repository {:#fetch-repository} + +The action of making a repo available on local disk by running its associated +repo rule. The repos defined in a workspace are not available on local disk +before they are fetched. + +Normally, Bazel only fetches a repo when it needs something from the repo, +and the repo hasn't already been fetched. If the repo has already been fetched +before, Bazel only re-fetches it if its definition has changed. + +### Directory layout {:#directory-layout} + +After being fetched, the repo can be found in the subdirectory `external` in the +[output base](/remote/output-directories), under its canonical name. + +You can run the following command to see the contents of the repo with the +canonical name `canonical_name`: + +```posix-terminal +ls $(bazel info output_base)/external/{{ '' }} canonical_name {{ '' }} +``` + +## Manage external dependencies with Bzlmod {:#bzlmod} + +Bzlmod, the new external dependency subsystem, does not directly work with repo +definitions. Instead, it builds a dependency graph from _modules_, runs +_extensions_ on top of the graph, and defines repos accordingly. + +A [Bazel **module**](/external/module) is a Bazel project that can have multiple +versions, each of which publishes metadata about other modules that it depends +on. A module must have a `MODULE.bazel` file at its repo root, next to the +`WORKSPACE` file. This file is the module's manifest, declaring its name, +version, list of dependencies, among other information. The following is a basic +example: + +```python +module(name = "my-module", version = "1.0") + +bazel_dep(name = "rules_cc", version = "0.0.1") +bazel_dep(name = "protobuf", version = "3.19.0") +``` + +A module must only list its direct dependencies, which Bzlmod looks up in a +[Bazel registry](/external/registry) — by default, the [Bazel Central +Registry](https://bcr.bazel.build/){:.external}. The registry provides the +dependencies' `MODULE.bazel` files, which allows Bazel to discover the entire +transitive dependency graph before performing version resolution. + +After version resolution, in which one version is selected for each module, +Bazel consults the registry again to learn how to define a repo for each module +(in most cases, using `http_archive`). + +Modules can also specify customized pieces of data called *tags*, which are +consumed by [**module extensions**](/external/extension) after module resolution +to define additional repos. These extensions have capabilities similar to repo +rules, enabling them to perform actions like file I/O and sending network +requests. Among other things, they allow Bazel to interact with other package +management systems while also respecting the dependency graph built out of Bazel +modules. + +### External links on Bzlmod {:#external-links} + +* [Bzlmod usage examples in bazelbuild/examples](https://github.com/bazelbuild/examples/tree/main/bzlmod){:.external} +* [Bazel External Dependencies Overhaul](https://docs.google.com/document/d/1moQfNcEIttsk6vYanNKIy3ZuK53hQUFq1b1r0rmsYVg/edit){: .external} + (original Bzlmod design doc) +* [BazelCon 2021 talk on Bzlmod](https://www.youtube.com/watch?v=TxOCKtU39Fs){: .external} +* [Bazel Community Day talk on Bzlmod](https://www.youtube.com/watch?v=MB6xxis9gWI){: .external} + +## Define repos with `WORKSPACE` {:#workspace-system} + +Historically, you can manage external dependencies by defining repos in the +`WORKSPACE` (or `WORKSPACE.bazel`) file. This file has a similar syntax to +`BUILD` files, employing repo rules instead of build rules. + +The following snippet is an example to use the `http_archive` repo rule in the +`WORKSPACE` file: + +```python +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +http_archive( + name = "foo", + urls = ["https://example.com/foo.zip"], + sha256 = "c9526390a7cd420fdcec2988b4f3626fe9c5b51e2959f685e8f4d170d1a9bd96", +) +``` + +The snippet defines a repo whose canonical name is `foo`. In the `WORKSPACE` +system, by default, the canonical name of a repo is also its apparent name to +all other repos. + + + +### Shortcomings of the `WORKSPACE` system {:#workspace-shortcomings} + +In the years since the `WORKSPACE` system was introduced, users have reported +many pain points, including: + +* Bazel does not evaluate the `WORKSPACE` files of any dependencies, so all + transitive dependencies must be defined in the `WORKSPACE` file of the main + repo, in addition to direct dependencies. +* To work around this, projects have adopted the "deps.bzl" pattern, in which + they define a macro which in turn defines multiple repos, and ask users to + call this macro in their `WORKSPACE` files. + * This has its own problems: macros cannot `load` other `.bzl` files, so + these projects have to define their transitive dependencies in this + "deps" macro, or work around this issue by having the user call multiple + layered "deps" macros. + * Bazel evaluates the `WORKSPACE` file sequentially. Additionally, + dependencies are specified using `http_archive` with URLs, without any + version information. This means that there is no reliable way to perform + version resolution in the case of diamond dependencies (`A` depends on + `B` and `C`; `B` and `C` both depend on different versions of `D`). + +Due to the shortcomings of WORKSPACE, Bzlmod is going to replace the legacy +WORKSPACE system in future Bazel releases. Please read the [Bzlmod migration +guide](/external/migration) on how to migrate to Bzlmod. \ No newline at end of file diff --git a/site/en/external/registry.md b/site/en/external/registry.md new file mode 100644 index 00000000000000..0cdf9d64b8b13d --- /dev/null +++ b/site/en/external/registry.md @@ -0,0 +1,116 @@ +Project: /_project.yaml +Book: /_book.yaml + +# Bazel registries + +{% include "_buttons.html" %} + +Bzlmod discovers dependencies by requesting their information from Bazel +*registries*: databases of Bazel modules. Currently, Bzlmod only supports +[*index registries*](#index_registry) — local directories or static HTTP servers +following a specific format. + +## Index registry + +An index registry is a local directory or a static HTTP server containing +information about a list of modules — including their homepage, maintainers, the +`MODULE.bazel` file of each version, and how to fetch the source of each +version. Notably, it does *not* need to serve the source archives itself. + +An index registry must follow the format below: + +* `/bazel_registry.json`: A JSON file containing metadata for the registry + like: + * `mirrors`: specifying the list of mirrors to use for source archives + * `module_base_path`: specifying the base path for modules with + `local_repository` type in the `source.json` file +* `/modules`: A directory containing a subdirectory for each module in this + registry +* `/modules/$MODULE`: A directory containing a subdirectory for each version + of this module, as well as: + * `metadata.json`: A JSON file containing information about the module, + with the following fields: + * `homepage`: The URL of the project's homepage + * `maintainers`: A list of JSON objects, each of which corresponds to + the information of a maintainer of the module *in the registry*. + Note that this is not necessarily the same as the *authors* of the + project + * `versions`: A list of all the versions of this module to be found in + this registry + * `yanked_versions`: A map of [*yanked* + versions](/external/module#yanked_versions) of this module. The keys + should be versions to yank and the values should be descriptions of + why the version is yanked, ideally containing a link to more + information +* `/modules/$MODULE/$VERSION`: A directory containing the following files: + * `MODULE.bazel`: The `MODULE.bazel` file of this module version + * `source.json`: A JSON file containing information on how to fetch the + source of this module version + * The default type is "archive", representing an `http_archive` repo, + with the following fields: + * `url`: The URL of the source archive + * `integrity`: The [Subresource + Integrity](https://w3c.github.io/webappsec-subresource-integrity/#integrity-metadata-description){: .external} + checksum of the archive + * `strip_prefix`: A directory prefix to strip when extracting the + source archive + * `patches`: A map containing patch files to apply to the + extracted archive. The patch files are located under the + `/modules/$MODULE/$VERSION/patches` directory. The keys are the + patch file names, and the values are the integrity checksum of + the patch files + * `patch_strip`: Same as the `--strip` argument of Unix `patch`. + * `archive_type`: The archive type of the downloaded file (Same as `type` on `http_archive`). + By default, the archive type is determined from the file extension of the URL. If the file has + no extension, you can explicitly specify one of the following: `"zip"`, `"jar"`, `"war"`, `"aar"`, + `"tar"`, `"tar.gz"`, `"tgz"`, `"tar.xz"`, `"txz"`, `"tar.zst"`, `"tzst"`, `tar.bz2`, `"ar"`, or `"deb"`. + * The type can be changed to use a local path, representing a + `local_repository` repo, with these fields: + * `type`: `local_path` + * `path`: The local path to the repo, calculated as following: + * If `path` is an absolute path, it stays as it is + * If `path` is a relative path and `module_base_path` is an + absolute path, it resolves to `/` + * If `path` and `module_base_path` are both relative paths, it + resolves to `//`. + Registry must be hosted locally and used by + `--registry=file://`. Otherwise, Bazel will + throw an error + * `patches/`: An optional directory containing patch files, only used when + `source.json` has "archive" type + +## Bazel Central Registry + +The Bazel Central Registry (BCR) at is an index +registry with contents backed by the GitHub repo +[`bazelbuild/bazel-central-registry`](https://github.com/bazelbuild/bazel-central-registry){: .external}. +You can browse its contents using the web frontend at +. + +The Bazel community maintains the BCR, and contributors are welcome to submit +pull requests. See the [BCR contribution +guidelines](https://github.com/bazelbuild/bazel-central-registry/blob/main/docs/README.md){: .external}. + +In addition to following the format of a normal index registry, the BCR requires +a `presubmit.yml` file for each module version +(`/modules/$MODULE/$VERSION/presubmit.yml`). This file specifies a few essential +build and test targets that you can use to check the validity of this module +version. The BCR's CI pipelines also uses this to ensure interoperability +between modules. + +## Selecting registries + +The repeatable Bazel flag `--registry` can be used to specify the list of +registries to request modules from, so you can set up your project to fetch +dependencies from a third-party or internal registry. Earlier registries take +precedence. For convenience, you can put a list of `--registry` flags in the +`.bazelrc` file of your project. + +If your registry is hosted on GitHub (for example, as a fork of +`bazelbuild/bazel-central-registry`) then your `--registry` value needs a raw +GitHub address under `raw.githubusercontent.com`. For example, on the `main` +branch of the `my-org` fork, you would set +`--registry=https://raw.githubusercontent.com/my-org/bazel-central-registry/main/`. + +Using the `--registry` flag stops the Bazel Central Registry from being used by +default, but you can add it back by adding `--registry=https://bcr.bazel.build`. diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 767069a0a2abdb..d6621ccab8948e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -50,7 +50,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionUsagesFunction; import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsUtil; import com.google.devtools.build.lib.bazel.commands.FetchCommand; -import com.google.devtools.build.lib.bazel.commands.ModqueryCommand; +import com.google.devtools.build.lib.bazel.commands.ModCommand; import com.google.devtools.build.lib.bazel.commands.SyncCommand; import com.google.devtools.build.lib.bazel.repository.LocalConfigPlatformFunction; import com.google.devtools.build.lib.bazel.repository.LocalConfigPlatformRule; @@ -200,7 +200,7 @@ public byte[] get( @Override public void serverInit(OptionsParsingResult startupOptions, ServerBuilder builder) { builder.addCommands(new FetchCommand()); - builder.addCommands(new ModqueryCommand()); + builder.addCommands(new ModCommand()); builder.addCommands(new SyncCommand()); builder.addInfoItems(new RepositoryCacheInfoItem(repositoryCache)); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java index 3b384a709ff8bd..cce77aa47fd3f5 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java @@ -15,6 +15,7 @@ package com.google.devtools.build.lib.bazel.bzlmod; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.errorprone.annotations.CanIgnoreReturnValue; @@ -85,6 +86,14 @@ public ArchiveRepoSpecBuilder setRemotePatchStrip(int remotePatchStrip) { return this; } + @CanIgnoreReturnValue + public ArchiveRepoSpecBuilder setArchiveType(String archiveType) { + if (!Strings.isNullOrEmpty(archiveType)) { + attrBuilder.put("type", archiveType); + } + return this; + } + public RepoSpec build() { return RepoSpec.builder() .setBzlFile("@bazel_tools//tools/build_defs/repo:http.bzl") diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 1ffb2157cacacd..1d5e560b921c7d 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -6,7 +6,9 @@ package( filegroup( name = "srcs", - srcs = glob(["*"]), + srcs = glob(["*"]) + [ + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand:srcs", + ], visibility = ["//src:__subpackages__"], ) @@ -256,6 +258,7 @@ java_library( ], deps = [ ":common", + ":module_extension", "//src/main/java/com/google/devtools/build/lib/skyframe:sky_functions", "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec:serialization-constant", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", @@ -272,6 +275,7 @@ java_library( deps = [ ":common", ":inspection", + ":module_extension", ":resolution", "//src/main/java/com/google/devtools/build/skyframe", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorFunction.java index c2c3ffe57a5751..cd6bee7fd142f9 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorFunction.java @@ -15,12 +15,15 @@ package com.google.devtools.build.lib.bazel.bzlmod; +import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSetMultimap; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule.ResolutionReason; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue; @@ -28,6 +31,7 @@ import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; +import com.google.devtools.build.skyframe.SkyframeLookupResult; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; @@ -49,6 +53,10 @@ public SkyValue compute(SkyKey skyKey, Environment env) if (root == null) { return null; } + BazelDepGraphValue depGraphValue = (BazelDepGraphValue) env.getValue(BazelDepGraphValue.KEY); + if (depGraphValue == null) { + return null; + } BazelModuleResolutionValue resolutionValue = (BazelModuleResolutionValue) env.getValue(BazelModuleResolutionValue.KEY); if (resolutionValue == null) { @@ -61,6 +69,12 @@ public SkyValue compute(SkyKey skyKey, Environment env) ImmutableMap depGraph = computeAugmentedGraph(unprunedDepGraph, resolvedDepGraph.keySet(), overrides); + ImmutableSetMultimap extensionToRepoInternalNames = + computeExtensionToRepoInternalNames(depGraphValue, env); + if (extensionToRepoInternalNames == null) { + return null; + } + // Group all ModuleKeys seen by their module name for easy lookup ImmutableMap> modulesIndex = ImmutableMap.copyOf( @@ -70,7 +84,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) AugmentedModule::getName, Collectors.mapping(AugmentedModule::getKey, toImmutableSet())))); - return BazelModuleInspectorValue.create(depGraph, modulesIndex); + return BazelModuleInspectorValue.create(depGraph, modulesIndex, extensionToRepoInternalNames); } public static ImmutableMap computeAugmentedGraph( @@ -157,4 +171,27 @@ public static ImmutableMap computeAugmentedGraph( return depGraphAugmentBuilder.entrySet().stream() .collect(toImmutableMap(Entry::getKey, e -> e.getValue().build())); } + + @Nullable + private ImmutableSetMultimap computeExtensionToRepoInternalNames( + BazelDepGraphValue depGraphValue, Environment env) throws InterruptedException { + ImmutableSet extensionEvalKeys = + depGraphValue.getExtensionUsagesTable().rowKeySet(); + ImmutableList singleEvalKeys = + extensionEvalKeys.stream().map(SingleExtensionEvalValue::key).collect(toImmutableList()); + SkyframeLookupResult singleEvalValues = env.getValuesAndExceptions(singleEvalKeys); + + ImmutableSetMultimap.Builder extensionToRepoInternalNames = + ImmutableSetMultimap.builder(); + for (SingleExtensionEvalValue.Key singleEvalKey : singleEvalKeys) { + SingleExtensionEvalValue singleEvalValue = + (SingleExtensionEvalValue) singleEvalValues.get(singleEvalKey); + if (singleEvalValue == null) { + return null; + } + extensionToRepoInternalNames.putAll( + singleEvalKey.argument(), singleEvalValue.getGeneratedRepoSpecs().keySet()); + } + return extensionToRepoInternalNames.build(); + } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorValue.java index 667f94e296dbdf..ef7242dc9acd13 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorValue.java @@ -19,6 +19,8 @@ import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.collect.ImmutableSortedMap; import com.google.devtools.build.lib.skyframe.SkyFunctions; import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; import com.google.devtools.build.skyframe.SkyKey; @@ -39,8 +41,10 @@ public abstract class BazelModuleInspectorValue implements SkyValue { public static BazelModuleInspectorValue create( ImmutableMap depGraph, - ImmutableMap> modulesIndex) { - return new AutoValue_BazelModuleInspectorValue(depGraph, modulesIndex); + ImmutableMap> modulesIndex, + ImmutableSetMultimap extensionToRepoInternalNames) { + return new AutoValue_BazelModuleInspectorValue( + depGraph, modulesIndex, extensionToRepoInternalNames); } /** @@ -58,6 +62,13 @@ public static BazelModuleInspectorValue create( */ public abstract ImmutableMap> getModulesIndex(); + /** + * A mapping from a module extension ID, to the list of "internal" names of the repos generated by + * that extension. The "internal" name is the name directly used by the extension when + * instantiating a repo rule. + */ + public abstract ImmutableSetMultimap getExtensionToRepoInternalNames(); + /** * A wrapper for {@link Module}, augmented with references to dependants (and also those who are * not used in the final dep graph). @@ -106,15 +117,18 @@ public abstract static class AugmentedModule { */ public abstract ImmutableMap getDepReasons(); - /** Shortcut for retrieving the union of both used and unused deps based on the unused flag. */ - public ImmutableMap getAllDeps(boolean unused) { + /** + * Shortcut for retrieving the sorted union of both used and unused deps based on the unused + * flag. + */ + public ImmutableSortedMap getAllDeps(boolean unused) { if (!unused) { - return getDeps().inverse(); + return ImmutableSortedMap.copyOf(getDeps().inverse(), ModuleKey.LEXICOGRAPHIC_COMPARATOR); } else { Map map = new HashMap<>(); map.putAll(getDeps().inverse()); map.putAll(getUnusedDeps().inverse()); - return ImmutableMap.copyOf(map); + return ImmutableSortedMap.copyOf(map, ModuleKey.LEXICOGRAPHIC_COMPARATOR); } } @@ -127,7 +141,7 @@ public ImmutableMap getAllDeps(boolean unused) { /** Flag for checking whether the module is present in the resolved dep graph. */ public boolean isUsed() { - return !getDependants().isEmpty(); + return getKey().equals(ModuleKey.ROOT) || !getDependants().isEmpty(); } /** Returns a new {@link AugmentedModule.Builder} with {@code key} set. */ @@ -196,17 +210,27 @@ public AugmentedModule.Builder addDepReason(String repoName, ResolutionReason re /** The reason why a final dependency of a module was resolved the way it was. */ public enum ResolutionReason { /** The dependency is the original dependency defined in the MODULE.bazel file. */ - ORIGINAL, + ORIGINAL(""), /** The dependency was replaced by the Minimal-Version Selection algorithm. */ - MINIMAL_VERSION_SELECTION, + MINIMAL_VERSION_SELECTION("MVS"), /** The dependency was replaced by a {@code single_version_override} rule. */ - SINGLE_VERSION_OVERRIDE, + SINGLE_VERSION_OVERRIDE("SVO"), /** The dependency was replaced by a {@code multiple_version_override} rule. */ - MULTIPLE_VERSION_OVERRIDE, + MULTIPLE_VERSION_OVERRIDE("MVO"), /** The dependency was replaced by one of the {@link NonRegistryOverride} rules. */ - ARCHIVE_OVERRIDE, - GIT_OVERRIDE, - LOCAL_PATH_OVERRIDE, + ARCHIVE_OVERRIDE("archive"), + GIT_OVERRIDE("git"), + LOCAL_PATH_OVERRIDE("local"); + + private final String label; + + ResolutionReason(String label) { + this.label = label; + } + + public String getLabel() { + return label; + } } } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java index 55ed3eacdb165f..2d7945f95f6eac 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java @@ -111,6 +111,7 @@ private static class SourceJson { Map patches; int patchStrip; String path; + String archiveType; } /** @@ -254,6 +255,7 @@ private RepoSpec createArchiveRepoSpec( .setStripPrefix(Strings.nullToEmpty(sourceJson.get().stripPrefix)) .setRemotePatches(remotePatches.buildOrThrow()) .setRemotePatchStrip(sourceJson.get().patchStrip) + .setArchiveType(sourceJson.get().archiveType) .build(); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionId.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionId.java index 8d667614832f8c..7448a0ced179ff 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionId.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionId.java @@ -15,17 +15,33 @@ package com.google.devtools.build.lib.bazel.bzlmod; +import static com.google.common.collect.Comparators.emptiesFirst; +import static com.google.common.primitives.Booleans.falseFirst; +import static java.util.Comparator.comparing; + import com.google.auto.value.AutoValue; import com.google.devtools.build.lib.cmdline.Label; +import java.util.Comparator; import java.util.Optional; /** A unique identifier for a {@link ModuleExtension}. */ @AutoValue public abstract class ModuleExtensionId { + public static final Comparator LEXICOGRAPHIC_COMPARATOR = + comparing(ModuleExtensionId::getBzlFileLabel) + .thenComparing(ModuleExtensionId::getExtensionName) + .thenComparing( + ModuleExtensionId::getIsolationKey, + emptiesFirst(IsolationKey.LEXICOGRAPHIC_COMPARATOR)); /** A unique identifier for a single isolated usage of a fixed module extension. */ @AutoValue abstract static class IsolationKey { + static final Comparator LEXICOGRAPHIC_COMPARATOR = + comparing(IsolationKey::getModule, ModuleKey.LEXICOGRAPHIC_COMPARATOR) + .thenComparing(IsolationKey::isDevUsage, falseFirst()) + .thenComparing(IsolationKey::getIsolatedUsageIndex); + /** The module which contains this isolated usage of a module extension. */ public abstract ModuleKey getModule(); @@ -54,4 +70,9 @@ public static ModuleExtensionId create( Label bzlFileLabel, String extensionName, Optional isolationKey) { return new AutoValue_ModuleExtensionId(bzlFileLabel, extensionName, isolationKey); } + + public String asTargetString() { + return String.format( + "%s%%%s", getBzlFileLabel().getUnambiguousCanonicalForm(), getExtensionName()); + } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java index d2333beec073ec..cce6323dca9002 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java @@ -752,9 +752,11 @@ public void singleVersionOverride( name = "multiple_version_override", doc = "Specifies that a dependency should still come from a registry, but multiple versions of" - + " it should be allowed to coexist. This directive only takes effect in the root" - + " module; in other words, if a module is used as a dependency by others, its own" - + " overrides are ignored.", + + " it should be allowed to coexist. See the documentation for" + + " more details. This" + + " directive only takes effect in the root module; in other words, if a module" + + " is used as a dependency by others, its own overrides are ignored.", parameters = { @Param( name = "module_name", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalValue.java index d8f6f631420f70..e0f68a43b3243f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalValue.java @@ -56,8 +56,12 @@ public static Key key(ModuleExtensionId id) { return Key.create(id); } + /** + * The {@link com.google.devtools.build.skyframe.SkyKey} of a {@link + * com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionEvalValue} + */ @AutoCodec - static class Key extends AbstractSkyKey { + public static class Key extends AbstractSkyKey { private static final Interner interner = BlazeInterners.newWeakInterner(); protected Key(ModuleExtensionId arg) { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/BUILD new file mode 100644 index 00000000000000..53d3a3d9ccbeac --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/BUILD @@ -0,0 +1,34 @@ +load("@rules_java//java:defs.bzl", "java_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//src:__subpackages__"], +) + +filegroup( + name = "srcs", + srcs = glob(["*"]), + visibility = ["//src:__subpackages__"], +) + +java_library( + name = "modcommand", + srcs = glob(["*.java"]), + deps = [ + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:module_extension", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:repo_rule_value", + "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/lib/packages", + "//src/main/java/com/google/devtools/build/lib/query2/query/output", + "//src/main/java/com/google/devtools/build/lib/util:maybe_complete_set", + "//src/main/java/com/google/devtools/common/options", + "//src/main/java/net/starlark/java/eval", + "//src/main/protobuf:failure_details_java_proto", + "//third_party:auto_value", + "//third_party:gson", + "//third_party:guava", + "//third_party:jsr305", + ], +) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ExtensionArg.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ExtensionArg.java new file mode 100644 index 00000000000000..b92d5f5eeda7ed --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ExtensionArg.java @@ -0,0 +1,147 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.bzlmod.modcommand; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableBiMap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionId; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModuleArg.ModuleArgConverter; +import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.cmdline.Label.RepoContext; +import com.google.devtools.build.lib.cmdline.LabelSyntaxException; +import com.google.devtools.build.lib.cmdline.RepositoryMapping; +import com.google.devtools.build.lib.server.FailureDetails.ModCommand.Code; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.Converters.CommaSeparatedNonEmptyOptionListConverter; +import com.google.devtools.common.options.OptionsParsingException; +import java.util.Optional; + +/** + * Represents a reference to a module extension, parsed from a command-line argument in the form of + * {@code %}. The {@code } part is parsed as a + * {@link ModuleArg}. Valid examples include {@code @rules_java//java:extensions.bzl%toolchains}, + * {@code rules_java@6.1.1//java:extensions.bzl%toolchains}, etc. + */ +@AutoValue +public abstract class ExtensionArg { + public static ExtensionArg create( + ModuleArg moduleArg, String repoRelativeBzlLabel, String extensionName) { + return new AutoValue_ExtensionArg(moduleArg, repoRelativeBzlLabel, extensionName); + } + + public abstract ModuleArg moduleArg(); + + public abstract String repoRelativeBzlLabel(); + + public abstract String extensionName(); + + /** Resolves this {@link ExtensionArg} to a {@link ModuleExtensionId}. */ + public final ModuleExtensionId resolveToExtensionId( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + ImmutableBiMap baseModuleDeps, + ImmutableBiMap baseModuleUnusedDeps) + throws InvalidArgumentException { + ImmutableSet refModules = + moduleArg() + .resolveToModuleKeys( + modulesIndex, + depGraph, + baseModuleDeps, + baseModuleUnusedDeps, + /* includeUnused= */ false, + /* warnUnused= */ false); + if (refModules.size() != 1) { + throw new InvalidArgumentException( + String.format( + "Module %s, as part of the extension specifier, should represent exactly one module" + + " version. Choose one of: %s.", + moduleArg(), refModules), + Code.INVALID_ARGUMENTS); + } + ModuleKey key = Iterables.getOnlyElement(refModules); + try { + Label label = + Label.parseWithRepoContext( + repoRelativeBzlLabel(), + RepoContext.of( + key.getCanonicalRepoName(), + // Intentionally allow no repo mapping here: it's a repo-relative label! + RepositoryMapping.create(ImmutableMap.of(), key.getCanonicalRepoName()))); + // TODO(wyv): support isolated extension usages? + return ModuleExtensionId.create(label, extensionName(), Optional.empty()); + } catch (LabelSyntaxException e) { + throw new InvalidArgumentException( + String.format("bad label format in %s: %s", repoRelativeBzlLabel(), e.getMessage()), + Code.INVALID_ARGUMENTS, + e); + } + } + + @Override + public final String toString() { + return moduleArg() + repoRelativeBzlLabel() + "%" + extensionName(); + } + + /** Converter for {@link ExtensionArg}. */ + public static class ExtensionArgConverter extends Converter.Contextless { + public static final ExtensionArgConverter INSTANCE = new ExtensionArgConverter(); + + @Override + public ExtensionArg convert(String input) throws OptionsParsingException { + int slashIdx = input.indexOf('/'); + if (slashIdx < 0) { + throw new OptionsParsingException("Invalid argument " + input + ": missing .bzl label"); + } + int percentIdx = input.indexOf('%'); + if (percentIdx < slashIdx) { + throw new OptionsParsingException("Invalid argument " + input + ": missing extension name"); + } + ModuleArg moduleArg = ModuleArgConverter.INSTANCE.convert(input.substring(0, slashIdx)); + return ExtensionArg.create( + moduleArg, input.substring(slashIdx, percentIdx), input.substring(percentIdx + 1)); + } + + @Override + public String getTypeDescription() { + return "an identifier in the format of %"; + } + } + + /** Converter for a comma-separated list of {@link ExtensionArg}s. */ + public static class CommaSeparatedExtensionArgListConverter + extends Converter.Contextless> { + + @Override + public ImmutableList convert(String input) throws OptionsParsingException { + ImmutableList args = new CommaSeparatedNonEmptyOptionListConverter().convert(input); + ImmutableList.Builder extensionArgs = new ImmutableList.Builder<>(); + for (String arg : args) { + extensionArgs.add(ExtensionArgConverter.INSTANCE.convert(arg)); + } + return extensionArgs.build(); + } + + @Override + public String getTypeDescription() { + return "a comma-separated list of s"; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/GraphvizOutputFormatter.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/GraphvizOutputFormatter.java new file mode 100644 index 00000000000000..254b5c89023ec4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/GraphvizOutputFormatter.java @@ -0,0 +1,167 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod.modcommand; + +import static com.google.common.collect.ImmutableSortedSet.toImmutableSortedSet; + +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionId; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; +import com.google.devtools.build.lib.bazel.bzlmod.Version; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModExecutor.ResultNode; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModExecutor.ResultNode.IsIndirect; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModExecutor.ResultNode.NodeMetadata; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModOptions.ExtensionShow; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.OutputFormatters.OutputFormatter; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.HashSet; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Set; + +/** + * Outputs graph-based results of {@link ModExecutor} in the Graphviz dot format which can be + * further pipelined to create an image graph visualization. + */ +public class GraphvizOutputFormatter extends OutputFormatter { + private StringBuilder str; + + @Override + public void output() { + str = new StringBuilder(); + str.append("digraph mygraph {\n") + .append(" ") + .append("node [ shape=box ]\n") + .append(" ") + .append("edge [ fontsize=8 ]\n"); + Set seen = new HashSet<>(); + Set seenExtensions = new HashSet<>(); + Deque toVisit = new ArrayDeque<>(); + seen.add(ModuleKey.ROOT); + toVisit.add(ModuleKey.ROOT); + + while (!toVisit.isEmpty()) { + ModuleKey key = toVisit.pop(); + AugmentedModule module = Objects.requireNonNull(depGraph.get(key)); + ResultNode node = Objects.requireNonNull(result.get(key)); + String sourceId = toId(key); + + if (key.equals(ModuleKey.ROOT)) { + String rootLabel = String.format(" (%s@%s)", module.getName(), module.getVersion()); + str.append(String.format(" \"\" [ label=\"%s\" ]\n", rootLabel)); + } else if (node.isTarget() || !module.isUsed()) { + String shapeString = node.isTarget() ? "diamond" : "box"; + String styleString = module.isUsed() ? "solid" : "dotted"; + str.append( + String.format(" %s [ shape=%s style=%s ]\n", toId(key), shapeString, styleString)); + } + + if (options.extensionInfo != ExtensionShow.HIDDEN) { + ImmutableSortedSet extensionsUsed = + extensionRepoImports.keySet().stream() + .filter(e -> extensionRepoImports.get(e).inverse().containsKey(key)) + .collect(toImmutableSortedSet(ModuleExtensionId.LEXICOGRAPHIC_COMPARATOR)); + for (ModuleExtensionId extensionId : extensionsUsed) { + if (options.extensionInfo == ExtensionShow.USAGES) { + str.append(String.format(" %s -> \"%s\"\n", toId(key), toId(extensionId))); + continue; + } + if (seenExtensions.add(extensionId)) { + printExtension(extensionId); + } + ImmutableSortedSet repoImports = + ImmutableSortedSet.copyOf(extensionRepoImports.get(extensionId).inverse().get(key)); + for (String repo : repoImports) { + str.append(String.format(" %s -> %s\n", toId(key), toId(extensionId, repo))); + } + } + } + for (Entry e : node.getChildrenSortedByKey()) { + ModuleKey childKey = e.getKey(); + IsIndirect childIndirect = e.getValue().isIndirect(); + String childId = toId(childKey); + if (childIndirect == IsIndirect.FALSE) { + String reasonLabel = getReasonLabel(childKey, key); + str.append(String.format(" %s -> %s [ %s ]\n", sourceId, childId, reasonLabel)); + } else { + str.append(String.format(" %s -> %s [ style=dashed ]\n", sourceId, childId)); + } + if (seen.add(childKey)) { + toVisit.add(childKey); + } + } + } + str.append("}"); + printer.println(str); + printer.flush(); + } + + private String toId(ModuleKey key) { + if (key.equals(ModuleKey.ROOT)) { + return "\"\""; + } + return String.format( + "\"%s@%s\"", + key.getName(), key.getVersion().equals(Version.EMPTY) ? "_" : key.getVersion()); + } + + private String toId(ModuleExtensionId id) { + return id.asTargetString(); + } + + private String toId(ModuleExtensionId id, String repo) { + return String.format("\"%s%%%s\"", toId(id), repo); + } + + private void printExtension(ModuleExtensionId id) { + str.append(String.format(" subgraph \"cluster_%s\" {\n", toId(id))); + str.append(String.format(" label=\"%s\"\n", toId(id))); + if (options.extensionInfo == ExtensionShow.USAGES) { + return; + } + ImmutableSortedSet usedRepos = + ImmutableSortedSet.copyOf(extensionRepoImports.get(id).keySet()); + for (String repo : usedRepos) { + str.append(String.format(" %s [ label=\"%s\" ]\n", toId(id, repo), repo)); + } + if (options.extensionInfo == ExtensionShow.REPOS) { + return; + } + ImmutableSortedSet unusedRepos = + ImmutableSortedSet.copyOf(Sets.difference(extensionRepos.get(id), usedRepos)); + for (String repo : unusedRepos) { + str.append(String.format(" %s [ label=\"%s\" style=dotted ]\n", toId(id, repo), repo)); + } + str.append(" }\n"); + } + + private String getReasonLabel(ModuleKey key, ModuleKey parent) { + if (!options.verbose) { + return ""; + } + Explanation explanation = getExtraResolutionExplanation(key, parent); + if (explanation == null) { + return ""; + } + String label = explanation.getResolutionReason().getLabel(); + if (!label.isEmpty()) { + return String.format("label=%s", label); + } + return ""; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/InvalidArgumentException.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/InvalidArgumentException.java new file mode 100644 index 00000000000000..ad7ea185396109 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/InvalidArgumentException.java @@ -0,0 +1,40 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod.modcommand; + +import com.google.devtools.build.lib.server.FailureDetails.ModCommand.Code; + +/** + * Exception thrown when a user-input argument is invalid (wrong number of arguments or the + * specified modules do not exist). + */ +public class InvalidArgumentException extends Exception { + private final Code code; + + public InvalidArgumentException(String message, Code code, Exception cause) { + super(message, cause); + this.code = code; + } + + public InvalidArgumentException(String message, Code code) { + super(message); + this.code = code; + } + + public Code getCode() { + return code; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/JsonOutputFormatter.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/JsonOutputFormatter.java new file mode 100644 index 00000000000000..1bc2d5bad259cb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/JsonOutputFormatter.java @@ -0,0 +1,162 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod.modcommand; + +import static com.google.common.collect.ImmutableSortedSet.toImmutableSortedSet; + +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionId; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModExecutor.ResultNode; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModExecutor.ResultNode.IsCycle; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModExecutor.ResultNode.IsExpanded; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModExecutor.ResultNode.IsIndirect; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModExecutor.ResultNode.NodeMetadata; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModOptions.ExtensionShow; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.OutputFormatters.OutputFormatter; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import java.util.HashSet; +import java.util.Map.Entry; +import java.util.Set; + +/** Outputs graph-based results of {@link ModExecutor} in JSON format. */ +public class JsonOutputFormatter extends OutputFormatter { + private Set seenExtensions; + + @Override + public void output() { + seenExtensions = new HashSet<>(); + JsonObject root = printModule(ModuleKey.ROOT, null, IsExpanded.TRUE, IsIndirect.FALSE); + root.addProperty("root", true); + printer.println(new GsonBuilder().setPrettyPrinting().create().toJson(root)); + } + + public String printKey(ModuleKey key) { + if (key.equals(ModuleKey.ROOT)) { + return ""; + } + return key.toString(); + } + + // Helper to print module extensions similarly to printModule + private JsonObject printExtension( + ModuleKey key, ModuleExtensionId extensionId, boolean unexpanded) { + JsonObject json = new JsonObject(); + json.addProperty("key", extensionId.asTargetString()); + json.addProperty("unexpanded", unexpanded); + if (options.extensionInfo == ExtensionShow.USAGES) { + return json; + } + ImmutableSortedSet repoImports = + ImmutableSortedSet.copyOf(extensionRepoImports.get(extensionId).inverse().get(key)); + JsonArray usedRepos = new JsonArray(); + for (String usedRepo : repoImports) { + usedRepos.add(usedRepo); + } + json.add("used_repos", usedRepos); + + if (unexpanded || options.extensionInfo == ExtensionShow.REPOS) { + return json; + } + ImmutableSortedSet unusedRepos = + ImmutableSortedSet.copyOf( + Sets.difference( + extensionRepos.get(extensionId), extensionRepoImports.get(extensionId).keySet())); + JsonArray unusedReposJson = new JsonArray(); + for (String unusedRepo : unusedRepos) { + unusedReposJson.add(unusedRepo); + } + json.add("unused_repos", unusedReposJson); + return json; + } + + // Depth-first traversal to display modules (while explicitly detecting cycles) + JsonObject printModule( + ModuleKey key, ModuleKey parent, IsExpanded expanded, IsIndirect indirect) { + ResultNode node = result.get(key); + AugmentedModule module = depGraph.get(key); + JsonObject json = new JsonObject(); + json.addProperty("key", printKey(key)); + if (!key.getName().equals(module.getName())) { + json.addProperty("name", module.getName()); + } + if (!key.getVersion().equals(module.getVersion())) { + json.addProperty("version", module.getVersion().toString()); + } + + if (indirect == IsIndirect.FALSE && options.verbose && parent != null) { + Explanation explanation = getExtraResolutionExplanation(key, parent); + if (explanation != null) { + if (!module.isUsed()) { + json.addProperty("unused", true); + json.addProperty("resolvedVersion", explanation.getChangedVersion().toString()); + } else { + json.addProperty("originalVersion", explanation.getChangedVersion().toString()); + } + json.addProperty("resolutionReason", explanation.getChangedVersion().toString()); + if (explanation.getRequestedByModules() != null) { + JsonArray requestedBy = new JsonArray(); + explanation.getRequestedByModules().forEach(k -> requestedBy.add(printKey(k))); + json.add("resolvedRequestedBy", requestedBy); + } + } + } + + if (expanded == IsExpanded.FALSE) { + json.addProperty("unexpanded", true); + return json; + } + + JsonArray deps = new JsonArray(); + JsonArray indirectDeps = new JsonArray(); + JsonArray cycles = new JsonArray(); + for (Entry e : node.getChildrenSortedByEdgeType()) { + ModuleKey childKey = e.getKey(); + IsExpanded childExpanded = e.getValue().isExpanded(); + IsIndirect childIndirect = e.getValue().isIndirect(); + IsCycle childCycles = e.getValue().isCycle(); + if (childCycles == IsCycle.TRUE) { + cycles.add(printModule(childKey, key, IsExpanded.FALSE, IsIndirect.FALSE)); + } else if (childIndirect == IsIndirect.TRUE) { + indirectDeps.add(printModule(childKey, key, childExpanded, IsIndirect.TRUE)); + } else { + deps.add(printModule(childKey, key, childExpanded, IsIndirect.FALSE)); + } + } + json.add("dependencies", deps); + json.add("indirectDependencies", indirectDeps); + json.add("cycles", cycles); + + if (options.extensionInfo == ExtensionShow.HIDDEN) { + return json; + } + ImmutableSortedSet extensionsUsed = + extensionRepoImports.keySet().stream() + .filter(e -> extensionRepoImports.get(e).inverse().containsKey(key)) + .collect(toImmutableSortedSet(ModuleExtensionId.LEXICOGRAPHIC_COMPARATOR)); + JsonArray extensionUsages = new JsonArray(); + for (ModuleExtensionId extensionId : extensionsUsed) { + boolean unexpandedExtension = !seenExtensions.add(extensionId); + extensionUsages.add(printExtension(key, extensionId, unexpandedExtension)); + } + json.add("extensionUsages", extensionUsages); + + return json; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModExecutor.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModExecutor.java new file mode 100644 index 00000000000000..9a84d0508f8d52 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModExecutor.java @@ -0,0 +1,641 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod.modcommand; + +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static com.google.common.collect.ImmutableSortedMap.toImmutableSortedMap; +import static com.google.common.collect.ImmutableSortedSet.toImmutableSortedSet; +import static java.util.Comparator.reverseOrder; +import static java.util.stream.Collectors.joining; + +import com.google.auto.value.AutoValue; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.ImmutableTable; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; +import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleValue; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionId; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionUsage; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; +import com.google.devtools.build.lib.bazel.bzlmod.Tag; +import com.google.devtools.build.lib.bazel.bzlmod.Version; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModExecutor.ResultNode.IsExpanded; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModExecutor.ResultNode.IsIndirect; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModExecutor.ResultNode.NodeMetadata; +import com.google.devtools.build.lib.packages.RawAttributeMapper; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.query2.query.output.BuildOutputFormatter.AttributeReader; +import com.google.devtools.build.lib.query2.query.output.BuildOutputFormatter.TargetOutputter; +import com.google.devtools.build.lib.query2.query.output.PossibleAttributeValues; +import com.google.devtools.build.lib.util.MaybeCompleteSet; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.Writer; +import java.util.ArrayDeque; +import java.util.Collections; +import java.util.Comparator; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import net.starlark.java.eval.Starlark; + +/** + * Executes inspection queries for {@link com.google.devtools.build.lib.bazel.commands.ModCommand} + * and prints the resulted output to the reporter's output stream using the different defined {@link + * OutputFormatters}. + */ +public class ModExecutor { + + private final ImmutableMap depGraph; + private final ImmutableTable extensionUsages; + private final ImmutableSetMultimap extensionRepos; + private final Optional> extensionFilter; + private final ModOptions options; + private final PrintWriter printer; + private ImmutableMap> + extensionRepoImports; + + public ModExecutor( + ImmutableMap depGraph, ModOptions options, Writer writer) { + this( + depGraph, + ImmutableTable.of(), + ImmutableSetMultimap.of(), + Optional.of(MaybeCompleteSet.completeSet()), + options, + writer); + } + + public ModExecutor( + ImmutableMap depGraph, + ImmutableTable extensionUsages, + ImmutableSetMultimap extensionRepos, + Optional> extensionFilter, + ModOptions options, + Writer writer) { + this.depGraph = depGraph; + this.extensionUsages = extensionUsages; + this.extensionRepos = extensionRepos; + this.extensionFilter = extensionFilter; + this.options = options; + this.printer = new PrintWriter(writer); + // Easier lookup table for repo imports by module. + // It is updated after pruneByDepthAndLink to filter out pruned modules. + this.extensionRepoImports = computeRepoImportsTable(depGraph.keySet()); + } + + public void graph(ImmutableSet from) { + ImmutableMap result = + expandAndPrune(from, computeExtensionFilterTargets(), false); + OutputFormatters.getFormatter(options.outputFormat) + .output(result, depGraph, extensionRepos, extensionRepoImports, printer, options); + } + + public void path(ImmutableSet from, ImmutableSet to) { + MaybeCompleteSet targets = + MaybeCompleteSet.unionElements(computeExtensionFilterTargets(), to); + ImmutableMap result = expandAndPrune(from, targets, true); + OutputFormatters.getFormatter(options.outputFormat) + .output(result, depGraph, extensionRepos, extensionRepoImports, printer, options); + } + + public void allPaths(ImmutableSet from, ImmutableSet to) { + MaybeCompleteSet targets = + MaybeCompleteSet.unionElements(computeExtensionFilterTargets(), to); + ImmutableMap result = expandAndPrune(from, targets, false); + OutputFormatters.getFormatter(options.outputFormat) + .output(result, depGraph, extensionRepos, extensionRepoImports, printer, options); + } + + public void showRepo(ImmutableMap targetRepoRuleValues) { + RuleDisplayOutputter outputter = new RuleDisplayOutputter(printer); + for (Entry e : targetRepoRuleValues.entrySet()) { + printer.printf("## %s:\n", e.getKey()); + outputter.outputRule(e.getValue().getRule()); + } + printer.flush(); + } + + public void showExtension( + ImmutableSet extensions, ImmutableSet fromUsages) { + for (ModuleExtensionId extension : extensions) { + displayExtension(extension, fromUsages); + } + printer.flush(); + } + + /** + * The core function which produces the {@link ResultNode} graph for all the graph-generating + * queries above. First, it expands the result graph starting from the {@code from} modules, up + * until the {@code to} target modules if they are specified. If {@code singlePath} is set, it + * will only contain a single path to one of the targets.
+ * Then it calls {@link ResultGraphPruner#pruneByDepth()} to prune nodes after the specified + * {@code depth} (root is at depth 0). If the query specifies any {@code to} targets, even if they + * are below the specified depth, they will still be included in the graph using some indirect + * (dotted) edges. If {@code from} nodes other than the root are specified, they will be pinned + * (connected directly under the root - using indirect edges if necessary). + */ + @VisibleForTesting + ImmutableMap expandAndPrune( + ImmutableSet from, MaybeCompleteSet targets, boolean singlePath) { + final MaybeCompleteSet coloredPaths = colorReversePathsToRoot(targets); + ImmutableMap.Builder resultBuilder = new ImmutableMap.Builder<>(); + ResultNode.Builder rootBuilder = ResultNode.builder(); + + ImmutableSet rootDirectChildren = + depGraph.get(ModuleKey.ROOT).getAllDeps(options.includeUnused).keySet(); + ImmutableSet rootPinnedChildren = + getPinnedChildrenOfRootInTheResultGraph(rootDirectChildren, from).stream() + .filter(coloredPaths::contains) + .filter(this::filterBuiltin) + .collect(toImmutableSortedSet(ModuleKey.LEXICOGRAPHIC_COMPARATOR)); + rootPinnedChildren.forEach( + moduleKey -> + rootBuilder.addChild( + moduleKey, + IsExpanded.TRUE, + rootDirectChildren.contains(moduleKey) ? IsIndirect.FALSE : IsIndirect.TRUE)); + resultBuilder.put(ModuleKey.ROOT, rootBuilder.build()); + + Set seen = new HashSet<>(rootPinnedChildren); + Deque toVisit = new ArrayDeque<>(rootPinnedChildren); + seen.add(ModuleKey.ROOT); + + while (!toVisit.isEmpty()) { + ModuleKey key = toVisit.pop(); + AugmentedModule module = depGraph.get(key); + ResultNode.Builder nodeBuilder = ResultNode.builder(); + nodeBuilder.setTarget(!targets.isComplete() && targets.contains(key)); + + ImmutableSortedSet moduleDeps = module.getAllDeps(options.includeUnused).keySet(); + for (ModuleKey childKey : moduleDeps) { + if (!coloredPaths.contains(childKey)) { + continue; + } + if (isBuiltin(childKey) && !options.includeBuiltin) { + continue; + } + if (seen.contains(childKey)) { + // Single paths should not contain cycles or unexpanded (duplicate) children + // TODO(andreisolo): Move the single path extraction to DFS otherwise it can produce a + // wrong answer in cycle edge-case A -> B -> C -> B with target D will not find ABD + // \__ D + if (!singlePath) { + nodeBuilder.addChild(childKey, IsExpanded.FALSE, IsIndirect.FALSE); + } + continue; + } + nodeBuilder.addChild(childKey, IsExpanded.TRUE, IsIndirect.FALSE); + seen.add(childKey); + toVisit.add(childKey); + if (singlePath) { + break; + } + } + + resultBuilder.put(key, nodeBuilder.build()); + } + return new ResultGraphPruner(targets, resultBuilder.buildOrThrow()).pruneByDepth(); + } + + private class ResultGraphPruner { + + private final Map oldResult; + private final Map resultBuilder; + private final Set parentStack; + private final MaybeCompleteSet targets; + + /** + * Constructs a ResultGraphPruner to prune the result graph after the specified depth. + * + * @param targets If not complete, it means that the result graph contains paths to some + * specific targets. This will cause some branches to contain, after the specified depths, + * some targets or target parents. As any other nodes omitted, transitive edges (embedding + * multiple edges) will be stored as indirect. + * @param oldResult The unpruned result graph. + */ + ResultGraphPruner(MaybeCompleteSet targets, Map oldResult) { + this.oldResult = oldResult; + this.resultBuilder = new HashMap<>(); + this.parentStack = new HashSet<>(); + this.targets = targets; + } + + /** + * Prunes the result tree after the specified depth using DFS (because some nodes may still + * appear after the max depth). + */ + private ImmutableMap pruneByDepth() { + ResultNode.Builder rootBuilder = ResultNode.builder(); + resultBuilder.put(ModuleKey.ROOT, rootBuilder); + + parentStack.add(ModuleKey.ROOT); + + for (Entry e : + oldResult.get(ModuleKey.ROOT).getChildrenSortedByKey()) { + rootBuilder.addChild(e.getKey(), IsExpanded.TRUE, e.getValue().isIndirect()); + visitVisible(e.getKey(), 1, ModuleKey.ROOT, IsExpanded.TRUE); + } + + // Build everything at the end to allow children to add themselves to their parent's + // adjacency list. + ImmutableMap result = + resultBuilder.entrySet().stream() + .collect( + toImmutableSortedMap( + ModuleKey.LEXICOGRAPHIC_COMPARATOR, + Entry::getKey, + e -> e.getValue().build())); + // Filter imports for nodes that were pruned during this process. + extensionRepoImports = computeRepoImportsTable(result.keySet()); + return result; + } + + // Handles graph traversal within the specified depth. + private void visitVisible( + ModuleKey moduleKey, int depth, ModuleKey parentKey, IsExpanded expanded) { + parentStack.add(moduleKey); + ResultNode oldNode = oldResult.get(moduleKey); + ResultNode.Builder nodeBuilder = + resultBuilder.computeIfAbsent(moduleKey, k -> ResultNode.builder()); + + nodeBuilder.setTarget(oldNode.isTarget()); + if (depth > 1) { + resultBuilder.get(parentKey).addChild(moduleKey, expanded, IsIndirect.FALSE); + } + + if (expanded == IsExpanded.FALSE) { + parentStack.remove(moduleKey); + return; + } + for (Entry e : oldNode.getChildrenSortedByKey()) { + ModuleKey childKey = e.getKey(); + IsExpanded childExpanded = e.getValue().isExpanded(); + if (notCycle(childKey)) { + if (depth < options.depth) { + visitVisible(childKey, depth + 1, moduleKey, childExpanded); + } else if (!targets.isComplete()) { + visitDetached(childKey, moduleKey, moduleKey, childExpanded); + } + } else if (options.cycles) { + nodeBuilder.addCycle(childKey); + } + } + parentStack.remove(moduleKey); + } + + // Detached mode is only present in withTargets and handles adding targets and target parents + // living below the specified depth to the graph. + private void visitDetached( + ModuleKey moduleKey, + ModuleKey parentKey, + ModuleKey lastVisibleParentKey, + IsExpanded expanded) { + parentStack.add(moduleKey); + ResultNode oldNode = oldResult.get(moduleKey); + ResultNode.Builder nodeBuilder = ResultNode.builder(); + nodeBuilder.setTarget(oldNode.isTarget()); + + if (oldNode.isTarget() || isTargetParent(oldNode)) { + ResultNode.Builder parentBuilder = resultBuilder.get(lastVisibleParentKey); + IsIndirect childIndirect = + lastVisibleParentKey.equals(parentKey) ? IsIndirect.FALSE : IsIndirect.TRUE; + parentBuilder.addChild(moduleKey, expanded, childIndirect); + resultBuilder.put(moduleKey, nodeBuilder); + lastVisibleParentKey = moduleKey; + } + + if (expanded == IsExpanded.FALSE) { + parentStack.remove(moduleKey); + return; + } + for (Entry e : oldNode.getChildrenSortedByKey()) { + ModuleKey childKey = e.getKey(); + IsExpanded childExpanded = e.getValue().isExpanded(); + if (notCycle(childKey)) { + visitDetached(childKey, moduleKey, lastVisibleParentKey, childExpanded); + } else if (options.cycles) { + nodeBuilder.addCycle(childKey); + } + } + parentStack.remove(moduleKey); + } + + private boolean notCycle(ModuleKey key) { + return !parentStack.contains(key); + } + + private boolean isTargetParent(ResultNode node) { + return node.getChildren().keys().stream() + .filter(Predicate.not(parentStack::contains)) + .anyMatch(targets::contains); + } + } + + /** + * Return a sorted list of modules that will be the direct children of the root in the result + * graph (original root's direct dependencies along with the specified targets). + */ + private ImmutableSortedSet getPinnedChildrenOfRootInTheResultGraph( + ImmutableSet rootDirectDeps, ImmutableSet fromTargets) { + Set targetKeys = new HashSet<>(fromTargets); + if (fromTargets.contains(ModuleKey.ROOT)) { + targetKeys.remove(ModuleKey.ROOT); + targetKeys.addAll(rootDirectDeps); + } + return ImmutableSortedSet.copyOf(ModuleKey.LEXICOGRAPHIC_COMPARATOR, targetKeys); + } + + private static boolean intersect( + MaybeCompleteSet a, Set b) { + if (a.isComplete()) { + return !b.isEmpty(); + } + return !Collections.disjoint(a.getElementsIfNotComplete(), b); + } + + /** + * If the extensionFilter option is set, computes the set of target modules that use the specified + * extension(s) and adds them to the list of specified targets if the query is a path(s) query. + */ + private MaybeCompleteSet computeExtensionFilterTargets() { + if (extensionFilter.isEmpty()) { + // If no --extension_filter is set, don't do anything here. + return MaybeCompleteSet.completeSet(); + } + return MaybeCompleteSet.copyOf( + depGraph.keySet().stream() + .filter(this::filterUnused) + .filter(this::filterBuiltin) + .filter(k -> intersect(extensionFilter.get(), extensionUsages.column(k).keySet())) + .collect(toImmutableSet())); + } + + /** + * Color all reverse paths from the target modules to the root so only modules which are part of + * these paths will be included in the output graph during the breadth-first traversal. + */ + private MaybeCompleteSet colorReversePathsToRoot(MaybeCompleteSet to) { + if (to.isComplete()) { + return MaybeCompleteSet.completeSet(); + } + + Set seen = new HashSet<>(to.getElementsIfNotComplete()); + Deque toVisit = new ArrayDeque<>(to.getElementsIfNotComplete()); + + while (!toVisit.isEmpty()) { + ModuleKey key = toVisit.pop(); + AugmentedModule module = depGraph.get(key); + Set parents = new HashSet<>(module.getDependants()); + if (options.includeUnused) { + parents.addAll(module.getOriginalDependants()); + } + for (ModuleKey parent : parents) { + if (isBuiltin(parent) && !options.includeBuiltin) { + continue; + } + if (seen.contains(parent)) { + continue; + } + seen.add(parent); + toVisit.add(parent); + } + } + + return MaybeCompleteSet.copyOf(seen); + } + + /** Compute the multimap of repo imports to modules for each extension. */ + private ImmutableMap> + computeRepoImportsTable(ImmutableSet presentModules) { + ImmutableMap.Builder> resultBuilder = + new ImmutableMap.Builder<>(); + for (ModuleExtensionId extension : extensionUsages.rowKeySet()) { + if (extensionFilter.isPresent() && !extensionFilter.get().contains(extension)) { + continue; + } + ImmutableSetMultimap.Builder modulesToImportsBuilder = + new ImmutableSetMultimap.Builder<>(); + for (Entry usage : + extensionUsages.rowMap().get(extension).entrySet()) { + if (!presentModules.contains(usage.getKey())) { + continue; + } + modulesToImportsBuilder.putAll(usage.getKey(), usage.getValue().getImports().values()); + } + resultBuilder.put(extension, modulesToImportsBuilder.build().inverse()); + } + return resultBuilder.buildOrThrow(); + } + + private boolean filterUnused(ModuleKey key) { + AugmentedModule module = depGraph.get(key); + return options.includeUnused || module.isUsed(); + } + + private boolean filterBuiltin(ModuleKey key) { + return options.includeBuiltin || !isBuiltin(key); + } + + /** Helper to display show_extension info. */ + private void displayExtension(ModuleExtensionId extension, ImmutableSet fromUsages) { + printer.printf("## %s:\n", extension.asTargetString()); + printer.println(); + printer.println("Fetched repositories:"); + // TODO(wyv): if `extension` doesn't exist, we crash. We should report a good error instead! + ImmutableSortedSet usedRepos = + ImmutableSortedSet.copyOf(extensionRepoImports.get(extension).keySet()); + ImmutableSortedSet unusedRepos = + ImmutableSortedSet.copyOf(Sets.difference(extensionRepos.get(extension), usedRepos)); + for (String repo : usedRepos) { + printer.printf( + " - %s (imported by %s)\n", + repo, + extensionRepoImports.get(extension).get(repo).stream() + .sorted(ModuleKey.LEXICOGRAPHIC_COMPARATOR) + .map(ModuleKey::toString) + .collect(joining(", "))); + } + for (String repo : unusedRepos) { + printer.printf(" - %s\n", repo); + } + printer.println(); + if (fromUsages.isEmpty()) { + fromUsages = ImmutableSet.copyOf(extensionUsages.rowMap().get(extension).keySet()); + } + for (ModuleKey module : fromUsages) { + if (!extensionUsages.contains(extension, module)) { + continue; + } + ModuleExtensionUsage usage = extensionUsages.get(extension, module); + printer.printf( + "## Usage in %s from %s:%s\n", + module, usage.getLocation().file(), usage.getLocation().line()); + for (Tag tag : usage.getTags()) { + printer.printf( + "%s.%s(%s)\n", + extension.getExtensionName(), + tag.getTagName(), + tag.getAttributeValues().attributes().entrySet().stream() + .map(e -> String.format("%s=%s", e.getKey(), Starlark.repr(e.getValue()))) + .collect(joining(", "))); + } + printer.printf("use_repo(\n"); + printer.printf(" %s,\n", extension.getExtensionName()); + for (Entry repo : usage.getImports().entrySet()) { + printer.printf( + " %s,\n", + repo.getKey().equals(repo.getValue()) + ? String.format("\"%s\"", repo.getKey()) + : String.format("%s=\"%s\"", repo.getKey(), repo.getValue())); + } + printer.printf(")\n\n"); + } + } + + private boolean isBuiltin(ModuleKey key) { + return key.equals(ModuleKey.create("bazel_tools", Version.EMPTY)) + || key.equals(ModuleKey.create("local_config_platform", Version.EMPTY)); + } + + /** A node representing a module that forms the result graph. */ + @AutoValue + public abstract static class ResultNode { + + /** Whether the module is one of the targets in a paths query. */ + abstract boolean isTarget(); + + enum IsExpanded { + FALSE, + TRUE + } + + enum IsIndirect { + FALSE, + TRUE + } + + enum IsCycle { + FALSE, + TRUE + } + + /** Detailed edge type for the {@link ResultNode} graph. */ + @AutoValue + public abstract static class NodeMetadata { + /** + * Whether the node should be expanded from this edge (the same node can appear in multiple + * places in a flattened graph). + */ + public abstract IsExpanded isExpanded(); + + /** Whether the edge is a direct edge or an indirect (transitive) one. */ + public abstract IsIndirect isIndirect(); + + /** Whether the edge is cycling back inside the flattened graph. */ + public abstract IsCycle isCycle(); + + private static NodeMetadata create( + IsExpanded isExpanded, IsIndirect isIndirect, IsCycle isCycle) { + return new AutoValue_ModExecutor_ResultNode_NodeMetadata(isExpanded, isIndirect, isCycle); + } + } + + /** List of children mapped to detailed edge types. */ + protected abstract ImmutableSetMultimap getChildren(); + + public ImmutableSortedSet> getChildrenSortedByKey() { + return ImmutableSortedSet.copyOf( + Entry.comparingByKey(ModuleKey.LEXICOGRAPHIC_COMPARATOR), getChildren().entries()); + } + + public ImmutableSortedSet> getChildrenSortedByEdgeType() { + return ImmutableSortedSet.copyOf( + Comparator., IsCycle>comparing( + e -> e.getValue().isCycle(), reverseOrder()) + .thenComparing(e -> e.getValue().isExpanded()) + .thenComparing(e -> e.getValue().isIndirect()) + .thenComparing(Entry::getKey, ModuleKey.LEXICOGRAPHIC_COMPARATOR), + getChildren().entries()); + } + + static ResultNode.Builder builder() { + return new AutoValue_ModExecutor_ResultNode.Builder().setTarget(false); + } + + @AutoValue.Builder + abstract static class Builder { + + abstract ResultNode.Builder setTarget(boolean value); + + abstract ImmutableSetMultimap.Builder childrenBuilder(); + + @CanIgnoreReturnValue + final Builder addChild(ModuleKey value, IsExpanded expanded, IsIndirect indirect) { + childrenBuilder().put(value, NodeMetadata.create(expanded, indirect, IsCycle.FALSE)); + return this; + } + + @CanIgnoreReturnValue + final Builder addCycle(ModuleKey value) { + childrenBuilder() + .put(value, NodeMetadata.create(IsExpanded.FALSE, IsIndirect.FALSE, IsCycle.TRUE)); + return this; + } + + abstract ResultNode build(); + } + } + + /** + * Uses Query's {@link TargetOutputter} to display the generating repo rule and other information. + */ + static class RuleDisplayOutputter { + private static final AttributeReader attrReader = + (rule, attr) -> + // Query's implementation copied + PossibleAttributeValues.forRuleAndAttribute( + rule, attr, /* mayTreatMultipleAsNone= */ true); + private final TargetOutputter targetOutputter; + private final PrintWriter printer; + + RuleDisplayOutputter(PrintWriter printer) { + this.printer = printer; + this.targetOutputter = + new TargetOutputter( + this.printer, + (rule, attr) -> RawAttributeMapper.of(rule).isConfigurable(attr.getName()), + "\n"); + } + + private void outputRule(Rule rule) { + try { + targetOutputter.outputRule(rule, attrReader, this.printer); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModOptions.java new file mode 100644 index 00000000000000..ab45297ae423fc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModOptions.java @@ -0,0 +1,261 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.bzlmod.modcommand; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.joining; + +import com.google.common.base.Ascii; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ExtensionArg.CommaSeparatedExtensionArgListConverter; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModuleArg.CommaSeparatedModuleArgListConverter; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModuleArg.ModuleArgConverter; +import com.google.devtools.common.options.EnumConverter; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionDocumentationCategory; +import com.google.devtools.common.options.OptionEffectTag; +import com.google.devtools.common.options.OptionsBase; + +/** Options for ModCommand */ +public class ModOptions extends OptionsBase { + + @Option( + name = "from", + defaultValue = "", + converter = CommaSeparatedModuleArgListConverter.class, + documentationCategory = OptionDocumentationCategory.MOD_COMMAND, + effectTags = {OptionEffectTag.TERMINAL_OUTPUT}, + help = + "The module(s) starting from which the dependency graph query will be displayed. Check" + + " each query’s description for the exact semantics. Defaults to .\n") + public ImmutableList modulesFrom; + + @Option( + name = "verbose", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.MOD_COMMAND, + effectTags = {OptionEffectTag.TERMINAL_OUTPUT}, + help = + "The queries will also display the reason why modules were resolved to their current" + + " version (if changed). Defaults to true only for the explain query.") + public boolean verbose; + + @Option( + name = "include_unused", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.MOD_COMMAND, + effectTags = {OptionEffectTag.TERMINAL_OUTPUT}, + help = + "The queries will also take into account and display the unused modules, which are not" + + " present in the module resolution graph after selection (due to the" + + " Minimal-Version Selection or override rules). This can have different effects for" + + " each of the query types i.e. include new paths in the all_paths command, or extra" + + " dependants in the explain command.") + public boolean includeUnused; + + @Option( + name = "extension_filter", + defaultValue = "null", + converter = CommaSeparatedExtensionArgListConverter.class, + documentationCategory = OptionDocumentationCategory.MOD_COMMAND, + effectTags = {OptionEffectTag.TERMINAL_OUTPUT}, + help = + "Only display the usages of these module extensions and the repos generated by them if" + + " their respective flags are set. If set, the result graph will only include paths" + + " that contain modules using the specified extensions. An empty list disables the" + + " filter, effectively specifying all possible extensions.") + public ImmutableList extensionFilter; + + @Option( + name = "extension_info", + defaultValue = "hidden", + converter = ExtensionShowConverter.class, + documentationCategory = OptionDocumentationCategory.MOD_COMMAND, + effectTags = {OptionEffectTag.TERMINAL_OUTPUT}, + help = + "Specify how much detail about extension usages to include in the query result." + + " \"Usages\" will only show the extensions names, \"repos\" will also include repos" + + " imported with use_repo, and \"all\" will also show the other repositories" + + " generated by extensions.\n") + public ExtensionShow extensionInfo; + + @Option( + name = "base_module", + defaultValue = "", + converter = ModuleArgConverter.class, + documentationCategory = OptionDocumentationCategory.MOD_COMMAND, + effectTags = {OptionEffectTag.TERMINAL_OUTPUT}, + help = "Specify a module relative to which the specified target repos will be interpreted.") + public ModuleArg baseModule; + + @Option( + name = "extension_usages", + defaultValue = "", + converter = CommaSeparatedModuleArgListConverter.class, + documentationCategory = OptionDocumentationCategory.MOD_COMMAND, + effectTags = {OptionEffectTag.TERMINAL_OUTPUT}, + help = + "Specify modules whose extension usages will be displayed in the show_extension query.") + public ImmutableList extensionUsages; + + @Option( + name = "depth", + defaultValue = "-1", + documentationCategory = OptionDocumentationCategory.MOD_COMMAND, + effectTags = {OptionEffectTag.TERMINAL_OUTPUT}, + help = + "Maximum display depth of the dependency tree. A depth of 1 displays the direct" + + " dependencies, for example. For tree, path and all_paths it defaults to" + + " Integer.MAX_VALUE, while for deps and explain it defaults to 1 (only displays" + + " direct deps of the root besides the target leaves and their parents).\n") + public int depth; + + @Option( + name = "cycles", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.MOD_COMMAND, + effectTags = {OptionEffectTag.TERMINAL_OUTPUT}, + help = + "Points out dependency cycles inside the displayed tree, which are normally ignored by" + + " default.") + public boolean cycles; + + @Option( + name = "include_builtin", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.MOD_COMMAND, + effectTags = {OptionEffectTag.TERMINAL_OUTPUT}, + help = + "Include built-in modules in the dependency graph. Disabled by default because it is" + + " quite noisy.") + public boolean includeBuiltin; + + @Option( + name = "charset", + defaultValue = "utf8", + converter = CharsetConverter.class, + documentationCategory = OptionDocumentationCategory.MOD_COMMAND, + effectTags = {OptionEffectTag.TERMINAL_OUTPUT}, + help = + "Chooses the character set to use for the tree. Only affects text output. Valid values" + + " are \"utf8\" or \"ascii\". Default is \"utf8\"") + public Charset charset; + + @Option( + name = "output", + defaultValue = "text", + converter = OutputFormatConverter.class, + documentationCategory = OptionDocumentationCategory.MOD_COMMAND, + effectTags = {OptionEffectTag.TERMINAL_OUTPUT}, + help = + "The format in which the query results should be printed. Allowed values for query are: " + + "text, json, graph") + public OutputFormat outputFormat; + + /** Possible subcommands that can be specified for the `mod` command. */ + public enum ModSubcommand { + DEPS(true), + GRAPH(true), + ALL_PATHS(true), + PATH(true), + EXPLAIN(true), + SHOW_REPO(false), + SHOW_EXTENSION(false); + + /** Whether this subcommand produces a graph output. */ + private final boolean isGraph; + + ModSubcommand(boolean isGraph) { + this.isGraph = isGraph; + } + + @Override + public String toString() { + return Ascii.toLowerCase(this.name()); + } + + public boolean isGraph() { + return isGraph; + } + + public static String printValues() { + return "(" + stream(values()).map(ModSubcommand::toString).collect(joining(", ")) + ")"; + } + } + + /** Converts a subcommand string to a properly typed {@link ModSubcommand} */ + public static class ModSubcommandConverter extends EnumConverter { + public ModSubcommandConverter() { + super(ModSubcommand.class, "mod subcommand"); + } + } + + enum ExtensionShow { + HIDDEN, + USAGES, + REPOS, + ALL + } + + /** Converts an option string to a properly typed {@link ExtensionShow} */ + public static class ExtensionShowConverter extends EnumConverter { + public ExtensionShowConverter() { + super(ExtensionShow.class, "extension show"); + } + } + + /** Charset to be used in outputting the `mod` command result. */ + public enum Charset { + UTF8, + ASCII + } + + /** Converts a charset option string to a properly typed {@link Charset} */ + public static class CharsetConverter extends EnumConverter { + public CharsetConverter() { + super(Charset.class, "output charset"); + } + } + + /** Possible formats of the `mod` command result. */ + public enum OutputFormat { + TEXT, + JSON, + GRAPH + } + + /** Converts an output format option string to a properly typed {@link OutputFormat} */ + public static class OutputFormatConverter extends EnumConverter { + public OutputFormatConverter() { + super(OutputFormat.class, "output format"); + } + } + + /** Returns a {@link ModOptions} filled with default values for testing. */ + static ModOptions getDefaultOptions() { + ModOptions options = new ModOptions(); + options.depth = Integer.MAX_VALUE; + options.cycles = false; + options.includeUnused = false; + options.verbose = false; + options.modulesFrom = + ImmutableList.of(ModuleArg.SpecificVersionOfModule.create(ModuleKey.ROOT)); + options.charset = Charset.UTF8; + options.outputFormat = OutputFormat.TEXT; + options.extensionFilter = null; + options.extensionInfo = ExtensionShow.HIDDEN; + return options; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModuleArg.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModuleArg.java new file mode 100644 index 00000000000000..9490f9bf5c81f8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModuleArg.java @@ -0,0 +1,418 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.bzlmod.modcommand; + +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.ImmutableSet.toImmutableSet; + +import com.google.auto.value.AutoValue; +import com.google.common.base.Ascii; +import com.google.common.collect.ImmutableBiMap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; +import com.google.devtools.build.lib.bazel.bzlmod.Version; +import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException; +import com.google.devtools.build.lib.cmdline.LabelSyntaxException; +import com.google.devtools.build.lib.cmdline.RepositoryMapping; +import com.google.devtools.build.lib.cmdline.RepositoryName; +import com.google.devtools.build.lib.server.FailureDetails.ModCommand.Code; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.Converters.CommaSeparatedNonEmptyOptionListConverter; +import com.google.devtools.common.options.OptionsParsingException; +import java.util.Optional; +import net.starlark.java.eval.EvalException; + +/** + * Represents a reference to one or more modules in the external dependency graph, used for + * modquery. This is parsed from a command-line argument (either as the value of a flag, or just as + * a bare argument), and can take one of various forms (see implementations). + */ +public interface ModuleArg { + + /** Resolves this module argument to a set of module keys. */ + ImmutableSet resolveToModuleKeys( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + ImmutableBiMap baseModuleDeps, + ImmutableBiMap baseModuleUnusedDeps, + boolean includeUnused, + boolean warnUnused) + throws InvalidArgumentException; + + /** Resolves this module argument to a set of repo names. */ + ImmutableMap resolveToRepoNames( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + RepositoryMapping mapping) + throws InvalidArgumentException; + + /** + * Refers to a specific version of a module. Parsed from {@code @}. {@code + * } can be the special string {@code _} to signify the empty version (for non-registry + * overrides). + */ + @AutoValue + abstract class SpecificVersionOfModule implements ModuleArg { + static SpecificVersionOfModule create(ModuleKey key) { + return new AutoValue_ModuleArg_SpecificVersionOfModule(key); + } + + public abstract ModuleKey moduleKey(); + + private void throwIfNonexistent( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + boolean includeUnused, + boolean warnUnused) + throws InvalidArgumentException { + AugmentedModule mod = depGraph.get(moduleKey()); + if (mod != null && !includeUnused && warnUnused && !mod.isUsed()) { + // Warn the user when unused modules are allowed and the specified version exists, but the + // --include_unused flag was not set. + throw new InvalidArgumentException( + String.format( + "Module version %s is unused as a result of module resolution. Use the" + + " --include_unused flag to include it.", + moduleKey()), + Code.INVALID_ARGUMENTS); + } + if (mod == null || (!includeUnused && !mod.isUsed())) { + ImmutableSet existingKeys = modulesIndex.get(moduleKey().getName()); + if (existingKeys == null) { + throw new InvalidArgumentException( + String.format( + "Module %s does not exist in the dependency graph.", moduleKey().getName()), + Code.INVALID_ARGUMENTS); + } + // If --include_unused is not true, unused modules will be considered non-existent and an + // error will be thrown. + ImmutableSet filteredKeys = + existingKeys.stream() + .filter(k -> includeUnused || depGraph.get(k).isUsed()) + .collect(toImmutableSet()); + throw new InvalidArgumentException( + String.format( + "Module version %s does not exist, available versions: %s.", + moduleKey(), filteredKeys), + Code.INVALID_ARGUMENTS); + } + } + + @Override + public final ImmutableSet resolveToModuleKeys( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + ImmutableBiMap baseModuleDeps, + ImmutableBiMap baseModuleUnusedDeps, + boolean includeUnused, + boolean warnUnused) + throws InvalidArgumentException { + throwIfNonexistent(modulesIndex, depGraph, includeUnused, warnUnused); + return ImmutableSet.of(moduleKey()); + } + + @Override + public ImmutableMap resolveToRepoNames( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + RepositoryMapping mapping) + throws InvalidArgumentException { + throwIfNonexistent( + modulesIndex, depGraph, /* includeUnused= */ false, /* warnUnused= */ false); + return ImmutableMap.of(moduleKey().toString(), moduleKey().getCanonicalRepoName()); + } + + @Override + public final String toString() { + return moduleKey().toString(); + } + } + + /** Refers to all versions of a module. Parsed from {@code }. */ + @AutoValue + abstract class AllVersionsOfModule implements ModuleArg { + static AllVersionsOfModule create(String moduleName) { + return new AutoValue_ModuleArg_AllVersionsOfModule(moduleName); + } + + public abstract String moduleName(); + + private ImmutableSet resolveInternal( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + boolean includeUnused, + boolean warnUnused) + throws InvalidArgumentException { + ImmutableSet existingKeys = modulesIndex.get(moduleName()); + if (existingKeys == null) { + throw new InvalidArgumentException( + String.format("Module %s does not exist in the dependency graph.", moduleName()), + Code.INVALID_ARGUMENTS); + } + ImmutableSet filteredKeys = + existingKeys.stream() + .filter(k -> includeUnused || depGraph.get(k).isUsed()) + .collect(toImmutableSet()); + if (filteredKeys.isEmpty()) { + if (warnUnused) { + throw new InvalidArgumentException( + String.format( + "Module %s is unused as a result of module resolution. Use the --include_unused" + + " flag to include it.", + moduleName()), + Code.INVALID_ARGUMENTS); + } + throw new InvalidArgumentException( + String.format("Module %s does not exist in the dependency graph.", moduleName()), + Code.INVALID_ARGUMENTS); + } + return filteredKeys; + } + + @Override + public ImmutableSet resolveToModuleKeys( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + ImmutableBiMap baseModuleDeps, + ImmutableBiMap baseModuleUnusedDeps, + boolean includeUnused, + boolean warnUnused) + throws InvalidArgumentException { + return resolveInternal(modulesIndex, depGraph, includeUnused, warnUnused); + } + + @Override + public ImmutableMap resolveToRepoNames( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + RepositoryMapping mapping) + throws InvalidArgumentException { + return resolveInternal( + modulesIndex, depGraph, /* includeUnused= */ false, /* warnUnused= */ false) + .stream() + .collect(toImmutableMap(ModuleKey::toString, ModuleKey::getCanonicalRepoName)); + } + + @Override + public final String toString() { + return moduleName(); + } + } + + /** + * Refers to a module with the given apparent repo name, in the context of {@code --base_module} + * (or when parsing that flag itself, in the context of the root module). Parsed from + * {@code @}. + */ + @AutoValue + abstract class ApparentRepoName implements ModuleArg { + static ApparentRepoName create(String name) { + return new AutoValue_ModuleArg_ApparentRepoName(name); + } + + public abstract String name(); + + @Override + public ImmutableSet resolveToModuleKeys( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + ImmutableBiMap baseModuleDeps, + ImmutableBiMap baseModuleUnusedDeps, + boolean includeUnused, + boolean warnUnused) + throws InvalidArgumentException { + ImmutableSet.Builder builder = new ImmutableSet.Builder<>(); + ModuleKey dep = baseModuleDeps.get(name()); + if (dep != null) { + builder.add(dep); + } + ModuleKey unusedDep = baseModuleUnusedDeps.get(name()); + if (includeUnused && unusedDep != null) { + builder.add(unusedDep); + } + var result = builder.build(); + if (result.isEmpty()) { + throw new InvalidArgumentException( + String.format( + "No module with the apparent repo name @%s exists in the dependency graph", name()), + Code.INVALID_ARGUMENTS); + } + return result; + } + + @Override + public ImmutableMap resolveToRepoNames( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + RepositoryMapping mapping) + throws InvalidArgumentException { + RepositoryName repoName = mapping.get(name()); + if (!repoName.isVisible()) { + throw new InvalidArgumentException( + String.format( + "No repo visible as %s from @%s", name(), repoName.getOwnerRepoDisplayString()), + Code.INVALID_ARGUMENTS); + } + return ImmutableMap.of(toString(), repoName); + } + + @Override + public final String toString() { + return "@" + name(); + } + } + + /** Refers to a module with the given canonical repo name. Parsed from {@code @@}. */ + @AutoValue + abstract class CanonicalRepoName implements ModuleArg { + static CanonicalRepoName create(RepositoryName repoName) { + return new AutoValue_ModuleArg_CanonicalRepoName(repoName); + } + + public abstract RepositoryName repoName(); + + @Override + public ImmutableSet resolveToModuleKeys( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + ImmutableBiMap baseModuleDeps, + ImmutableBiMap baseModuleUnusedDeps, + boolean includeUnused, + boolean warnUnused) + throws InvalidArgumentException { + Optional mod = + depGraph.values().stream() + .filter(m -> m.getKey().getCanonicalRepoName().equals(repoName())) + .findAny(); + if (mod.isPresent() && !includeUnused && warnUnused && !mod.get().isUsed()) { + // Warn the user when unused modules are allowed and the specified version exists, but the + // --include_unused flag was not set. + throw new InvalidArgumentException( + String.format( + "Module version %s is unused as a result of module resolution. Use the" + + " --include_unused flag to include it.", + mod.get().getKey()), + Code.INVALID_ARGUMENTS); + } + if (mod.isEmpty() || (!includeUnused && !mod.get().isUsed())) { + // If --include_unused is not true, unused modules will be considered non-existent and an + // error will be thrown. + throw new InvalidArgumentException( + String.format( + "No module with the canonical repo name @@%s exists in the dependency graph", + repoName().getName()), + Code.INVALID_ARGUMENTS); + } + return ImmutableSet.of(mod.get().getKey()); + } + + @Override + public ImmutableMap resolveToRepoNames( + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + RepositoryMapping mapping) + throws InvalidArgumentException { + if (depGraph.values().stream() + .filter(m -> m.getKey().getCanonicalRepoName().equals(repoName()) && m.isUsed()) + .findAny() + .isEmpty()) { + throw new InvalidArgumentException( + String.format( + "No module with the canonical repo name @@%s exists in the dependency graph", + repoName().getName()), + Code.INVALID_ARGUMENTS); + } + return ImmutableMap.of(toString(), repoName()); + } + + @Override + public final String toString() { + return "@@" + repoName().getName(); + } + } + + /** Converter for {@link ModuleArg}. */ + final class ModuleArgConverter extends Converter.Contextless { + public static final ModuleArgConverter INSTANCE = new ModuleArgConverter(); + + @Override + public ModuleArg convert(String input) throws OptionsParsingException { + if (Ascii.equalsIgnoreCase(input, "")) { + return SpecificVersionOfModule.create(ModuleKey.ROOT); + } + if (input.startsWith("@@")) { + try { + return CanonicalRepoName.create(RepositoryName.create(input.substring(2))); + } catch (LabelSyntaxException e) { + throw new OptionsParsingException("invalid argument '" + input + "': " + e.getMessage()); + } + } + if (input.startsWith("@")) { + String apparentRepoName = input.substring(1); + try { + RepositoryName.validateUserProvidedRepoName(apparentRepoName); + } catch (EvalException e) { + throw new OptionsParsingException("invalid argument '" + input + "': " + e.getMessage()); + } + return ApparentRepoName.create(apparentRepoName); + } + int atIdx = input.indexOf('@'); + if (atIdx >= 0) { + String moduleName = input.substring(0, atIdx); + String versionStr = input.substring(atIdx + 1); + if (versionStr.isEmpty()) { + throw new OptionsParsingException( + "invalid argument '" + input + "': use _ for the empty version"); + } + try { + Version version = versionStr.equals("_") ? Version.EMPTY : Version.parse(versionStr); + return SpecificVersionOfModule.create(ModuleKey.create(moduleName, version)); + } catch (ParseException e) { + throw new OptionsParsingException("invalid argument '" + input + "': " + e.getMessage()); + } + } + return AllVersionsOfModule.create(input); + } + + @Override + public String getTypeDescription() { + return "\"\" for the root module; @ for a specific version of a" + + " module; for all versions of a module; @ for a repo with the" + + " given apparent name; or @@ for a repo with the given canonical name"; + } + } + + /** Converter for a comma-separated list of {@link ModuleArg}s. */ + class CommaSeparatedModuleArgListConverter + extends Converter.Contextless> { + + @Override + public ImmutableList convert(String input) throws OptionsParsingException { + ImmutableList args = new CommaSeparatedNonEmptyOptionListConverter().convert(input); + ImmutableList.Builder moduleArgs = new ImmutableList.Builder<>(); + for (String arg : args) { + moduleArgs.add(ModuleArgConverter.INSTANCE.convert(arg)); + } + return moduleArgs.build(); + } + + @Override + public String getTypeDescription() { + return "a comma-separated list of s"; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/OutputFormatters.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/OutputFormatters.java new file mode 100644 index 00000000000000..3411a1c2acd282 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/OutputFormatters.java @@ -0,0 +1,174 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod.modcommand; + +import static java.util.stream.Collectors.joining; + +import com.google.auto.value.AutoValue; +import com.google.common.base.Ascii; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule.ResolutionReason; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionId; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; +import com.google.devtools.build.lib.bazel.bzlmod.Version; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModExecutor.ResultNode; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModOptions.OutputFormat; +import java.io.PrintWriter; +import javax.annotation.Nullable; + +/** + * Contains the output formatters for the graph-based results of {@link ModExecutor} that can be + * specified using {@link ModOptions#outputFormat}. + */ +public final class OutputFormatters { + + private static final OutputFormatter textFormatter = new TextOutputFormatter(); + private static final OutputFormatter jsonFormatter = new JsonOutputFormatter(); + private static final OutputFormatter graphvizFormatter = new GraphvizOutputFormatter(); + + private OutputFormatters() {} + + static OutputFormatter getFormatter(OutputFormat format) { + switch (format) { + case TEXT: + return textFormatter; + case JSON: + return jsonFormatter; + case GRAPH: + return graphvizFormatter; + } + throw new IllegalArgumentException("Output format cannot be null."); + } + + abstract static class OutputFormatter { + + protected ImmutableMap result; + protected ImmutableMap depGraph; + protected ImmutableSetMultimap extensionRepos; + protected ImmutableMap> + extensionRepoImports; + protected PrintWriter printer; + protected ModOptions options; + + /** Compact representation of the data provided by the {@code --verbose} flag. */ + @AutoValue + abstract static class Explanation { + + /** The version from/to which the module was changed after resolution. */ + abstract Version getChangedVersion(); + + abstract ResolutionReason getResolutionReason(); + + /** + * The list of modules who originally requested the selected version in the case of + * Minimal-Version-Selection. + */ + @Nullable + abstract ImmutableSet getRequestedByModules(); + + static Explanation create( + Version version, ResolutionReason reason, ImmutableSet requestedByModules) { + return new AutoValue_OutputFormatters_OutputFormatter_Explanation( + version, reason, requestedByModules); + } + + /** + * Gets the exact label that is printed next to the module if the {@code --verbose} flag is + * enabled. + */ + String toExplanationString(boolean unused) { + String changedVersionLabel = + getChangedVersion().equals(Version.EMPTY) ? "_" : getChangedVersion().toString(); + String toOrWasString = unused ? "to" : "was"; + String reasonString = + getRequestedByModules() != null + ? getRequestedByModules().stream().map(ModuleKey::toString).collect(joining(", ")) + : Ascii.toLowerCase(getResolutionReason().toString()); + return String.format("(%s %s, cause %s)", toOrWasString, changedVersionLabel, reasonString); + } + } + + /** Exposed API of the formatter during which the necessary objects are injected. */ + void output( + ImmutableMap result, + ImmutableMap depGraph, + ImmutableSetMultimap extensionRepos, + ImmutableMap> + extensionRepoImports, + PrintWriter printer, + ModOptions options) { + this.result = result; + this.depGraph = depGraph; + this.extensionRepos = extensionRepos; + this.extensionRepoImports = extensionRepoImports; + this.printer = printer; + this.options = options; + output(); + printer.flush(); + } + + /** Internal implementation of the formatter output function. */ + protected abstract void output(); + + /** + * Exists only for testing, because normally the depGraph and options are injected inside the + * public API call. + */ + protected Explanation getExtraResolutionExplanation( + ModuleKey key, + ModuleKey parent, + ImmutableMap depGraph, + ModOptions options) { + this.depGraph = depGraph; + this.options = options; + return getExtraResolutionExplanation(key, parent); + } + + /** + * Returns {@code null} if the module version has not changed during resolution or if the module + * is <root>. + */ + @Nullable + protected Explanation getExtraResolutionExplanation(ModuleKey key, ModuleKey parent) { + if (key.equals(ModuleKey.ROOT)) { + return null; + } + AugmentedModule module = depGraph.get(key); + AugmentedModule parentModule = depGraph.get(parent); + String repoName = parentModule.getAllDeps(options.includeUnused).get(key); + Version changedVersion; + ImmutableSet changedByModules = null; + ResolutionReason reason = parentModule.getDepReasons().get(repoName); + AugmentedModule replacement = + module.isUsed() ? module : depGraph.get(parentModule.getDeps().get(repoName)); + if (reason != ResolutionReason.ORIGINAL) { + if (!module.isUsed()) { + changedVersion = replacement.getVersion(); + } else { + AugmentedModule old = depGraph.get(parentModule.getUnusedDeps().get(repoName)); + changedVersion = old.getVersion(); + } + if (reason == ResolutionReason.MINIMAL_VERSION_SELECTION) { + changedByModules = replacement.getOriginalDependants(); + } + return Explanation.create(changedVersion, reason, changedByModules); + } + return null; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/TextOutputFormatter.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/TextOutputFormatter.java new file mode 100644 index 00000000000000..7413e42ff58861 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/TextOutputFormatter.java @@ -0,0 +1,246 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod.modcommand; + +import static com.google.common.collect.ImmutableSortedSet.toImmutableSortedSet; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionId; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; +import com.google.devtools.build.lib.bazel.bzlmod.Version; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModExecutor.ResultNode; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModExecutor.ResultNode.IsCycle; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModExecutor.ResultNode.IsExpanded; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModExecutor.ResultNode.IsIndirect; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModExecutor.ResultNode.NodeMetadata; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModOptions.Charset; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModOptions.ExtensionShow; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.OutputFormatters.OutputFormatter; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Set; + +/** Outputs graph-based results of {@link ModExecutor} in a human-readable text format. */ +public class TextOutputFormatter extends OutputFormatter { + + private Deque isLastChildStack; + private DrawCharset drawCharset; + private Set seenExtensions; + private StringBuilder str; + + @Override + public void output() { + if (options.charset == Charset.ASCII) { + drawCharset = DrawCharset.ASCII; + } else { + drawCharset = DrawCharset.UTF8; + } + isLastChildStack = new ArrayDeque<>(); + seenExtensions = new HashSet<>(); + str = new StringBuilder(); + printModule(ModuleKey.ROOT, null, IsExpanded.TRUE, IsIndirect.FALSE, IsCycle.FALSE, 0); + this.printer.println(str); + } + + // Prints the indents and the tree drawing characters. + private void printTreeDrawing(IsIndirect indirect, int depth) { + if (depth > 0) { + int indents = isLastChildStack.size() - 1; + Iterator value = isLastChildStack.descendingIterator(); + for (int i = 0; i < indents; i++) { + boolean isLastChild = value.next(); + if (isLastChild) { + str.append(drawCharset.emptyIndent); + } else { + str.append(drawCharset.prevChildIndent); + } + } + if (indirect == IsIndirect.TRUE) { + if (isLastChildStack.getFirst()) { + str.append(drawCharset.lastIndirectChildIndent); + } else { + str.append(drawCharset.indirectChildIndent); + } + } else { + if (isLastChildStack.getFirst()) { + str.append(drawCharset.lastChildIndent); + + } else { + str.append(drawCharset.childIndent); + } + } + } + } + + // Helper to print module extensions similarly to printModule. + private void printExtension( + ModuleKey key, ModuleExtensionId extensionId, boolean unexpanded, int depth) { + printTreeDrawing(IsIndirect.FALSE, depth); + str.append('$'); + str.append(extensionId.asTargetString()); + str.append(' '); + if (unexpanded && options.extensionInfo == ExtensionShow.ALL) { + str.append("... "); + } + str.append("\n"); + if (options.extensionInfo == ExtensionShow.USAGES) { + return; + } + ImmutableSortedSet repoImports = + ImmutableSortedSet.copyOf(extensionRepoImports.get(extensionId).inverse().get(key)); + ImmutableSortedSet unusedRepos = ImmutableSortedSet.of(); + if (!unexpanded && options.extensionInfo == ExtensionShow.ALL) { + unusedRepos = + ImmutableSortedSet.copyOf( + Sets.difference( + extensionRepos.get(extensionId), extensionRepoImports.get(extensionId).keySet())); + } + int totalChildrenNum = repoImports.size() + unusedRepos.size(); + int currChild = 1; + for (String usedRepo : repoImports) { + isLastChildStack.push(currChild++ == totalChildrenNum); + printExtensionRepo(usedRepo, IsIndirect.FALSE, depth + 1); + isLastChildStack.pop(); + } + if (unexpanded || options.extensionInfo == ExtensionShow.REPOS) { + return; + } + for (String unusedPackage : unusedRepos) { + isLastChildStack.push(currChild++ == totalChildrenNum); + printExtensionRepo(unusedPackage, IsIndirect.TRUE, depth + 1); + isLastChildStack.pop(); + } + } + + // Prints an extension repo line. + private void printExtensionRepo(String repoName, IsIndirect indirectLink, int depth) { + printTreeDrawing(indirectLink, depth); + str.append(repoName).append("\n"); + } + + // Depth-first traversal to print the actual output + private void printModule( + ModuleKey key, + ModuleKey parent, + IsExpanded expanded, + IsIndirect indirect, + IsCycle cycle, + int depth) { + printTreeDrawing(indirect, depth); + + ResultNode node = Objects.requireNonNull(result.get(key)); + if (key.equals(ModuleKey.ROOT)) { + AugmentedModule rootModule = depGraph.get(ModuleKey.ROOT); + Preconditions.checkNotNull(rootModule); + str.append( + String.format( + " (%s@%s)", + rootModule.getName(), + rootModule.getVersion().equals(Version.EMPTY) ? "_" : rootModule.getVersion())); + } else { + str.append(key).append(" "); + } + + int totalChildrenNum = node.getChildren().size(); + + ImmutableSortedSet extensionsUsed = + extensionRepoImports.keySet().stream() + .filter(e -> extensionRepoImports.get(e).inverse().containsKey(key)) + .collect(toImmutableSortedSet(ModuleExtensionId.LEXICOGRAPHIC_COMPARATOR)); + if (options.extensionInfo != ExtensionShow.HIDDEN) { + totalChildrenNum += extensionsUsed.size(); + } + + if (cycle == IsCycle.TRUE) { + str.append("(cycle) "); + } else if (expanded == IsExpanded.FALSE) { + str.append("(*) "); + } else { + if (totalChildrenNum != 0 && node.isTarget()) { + str.append("# "); + } + } + AugmentedModule module = Objects.requireNonNull(depGraph.get(key)); + if (!options.verbose && !module.isUsed()) { + str.append("(unused) "); + } + // If the edge is indirect, the parent is not only unknown, but the node could have come + // from multiple paths merged in the process, so we skip the resolution explanation. + if (indirect == IsIndirect.FALSE && options.verbose && parent != null) { + Explanation explanation = getExtraResolutionExplanation(key, parent); + if (explanation != null) { + str.append(explanation.toExplanationString(!module.isUsed())); + } + } + + str.append("\n"); + + if (expanded == IsExpanded.FALSE) { + return; + } + + int currChild = 1; + if (options.extensionInfo != ExtensionShow.HIDDEN) { + for (ModuleExtensionId extensionId : extensionsUsed) { + boolean unexpandedExtension = !seenExtensions.add(extensionId); + isLastChildStack.push(currChild++ == totalChildrenNum); + printExtension(key, extensionId, unexpandedExtension, depth + 1); + isLastChildStack.pop(); + } + } + for (Entry e : node.getChildrenSortedByEdgeType()) { + ModuleKey childKey = e.getKey(); + IsExpanded childExpanded = e.getValue().isExpanded(); + IsIndirect childIndirect = e.getValue().isIndirect(); + IsCycle childCycles = e.getValue().isCycle(); + isLastChildStack.push(currChild++ == totalChildrenNum); + printModule(childKey, key, childExpanded, childIndirect, childCycles, depth + 1); + isLastChildStack.pop(); + } + } + + enum DrawCharset { + ASCII(" ", "| ", "|___", "|...", "|___", "|..."), + UTF8(" ", "│ ", "├───", "├╌╌╌", "└───", "└╌╌╌"); + final String emptyIndent; + final String prevChildIndent; + final String childIndent; + final String indirectChildIndent; + final String lastChildIndent; + final String lastIndirectChildIndent; + + DrawCharset( + String emptyIndent, + String prevChildIndent, + String childIndent, + String indirectChildIndent, + String lastChildIndent, + String lastIndirectChildIndent) { + this.emptyIndent = emptyIndent; + this.prevChildIndent = prevChildIndent; + this.childIndent = childIndent; + this.indirectChildIndent = indirectChildIndent; + this.lastChildIndent = lastChildIndent; + this.lastIndirectChildIndent = lastIndirectChildIndent; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD index e6d63e1b2a042e..07f59db3e08e0e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD @@ -17,7 +17,7 @@ java_library( ), resources = [ "fetch.txt", - "modquery.txt", + "mod.txt", "sync.txt", ], deps = [ @@ -30,7 +30,10 @@ java_library( "//src/main/java/com/google/devtools/build/lib/bazel:resolved_event", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:module_extension", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:repo_rule_value", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution", + "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand", "//src/main/java/com/google/devtools/build/lib/bazel/repository", "//src/main/java/com/google/devtools/build/lib/bazel/repository/starlark", "//src/main/java/com/google/devtools/build/lib/cmdline", @@ -51,14 +54,13 @@ java_library( "//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code", "//src/main/java/com/google/devtools/build/lib/util:exit_code", "//src/main/java/com/google/devtools/build/lib/util:interrupted_failure_details", + "//src/main/java/com/google/devtools/build/lib/util:maybe_complete_set", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/skyframe", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", "//src/main/java/com/google/devtools/common/options", "//src/main/java/net/starlark/java/eval", "//src/main/protobuf:failure_details_java_proto", - "//third_party:auto_value", "//third_party:guava", - "//third_party:jsr305", ], ) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java new file mode 100644 index 00000000000000..7f4b9676f1fbb9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java @@ -0,0 +1,497 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.bazel.commands; + +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModOptions.Charset.UTF8; +import static java.nio.charset.StandardCharsets.US_ASCII; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.stream.Collectors.joining; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableBiMap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphValue; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; +import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleValue; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionId; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ExtensionArg; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ExtensionArg.ExtensionArgConverter; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.InvalidArgumentException; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModExecutor; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModOptions; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModOptions.ModSubcommand; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModOptions.ModSubcommandConverter; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModuleArg; +import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModuleArg.ModuleArgConverter; +import com.google.devtools.build.lib.cmdline.RepositoryMapping; +import com.google.devtools.build.lib.cmdline.RepositoryName; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.pkgcache.PackageOptions; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandResult; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.runtime.KeepGoingOption; +import com.google.devtools.build.lib.runtime.LoadingPhaseThreadsOption; +import com.google.devtools.build.lib.server.FailureDetails; +import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; +import com.google.devtools.build.lib.server.FailureDetails.ModCommand.Code; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.DetailedExitCode; +import com.google.devtools.build.lib.util.InterruptedFailureDetails; +import com.google.devtools.build.lib.util.MaybeCompleteSet; +import com.google.devtools.build.skyframe.EvaluationContext; +import com.google.devtools.build.skyframe.EvaluationResult; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import com.google.devtools.common.options.OptionPriority.PriorityCategory; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsParsingException; +import com.google.devtools.common.options.OptionsParsingResult; +import java.io.OutputStreamWriter; +import java.util.List; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; + +/** Queries the Bzlmod external dependency graph. */ +@Command( + name = ModCommand.NAME, + // TODO(andreisolo): figure out which extra options are really needed + options = { + ModOptions.class, + PackageOptions.class, + KeepGoingOption.class, + LoadingPhaseThreadsOption.class + }, + help = "resource:mod.txt", + shortDescription = "Queries the Bzlmod external dependency graph", + allowResidue = true) +public final class ModCommand implements BlazeCommand { + + public static final String NAME = "mod"; + + @Override + public void editOptions(OptionsParser optionsParser) { + try { + optionsParser.parse( + PriorityCategory.SOFTWARE_REQUIREMENT, + "Option required by the mod command", + ImmutableList.of("--enable_bzlmod")); + } catch (OptionsParsingException e) { + // Should never happen. + throw new IllegalStateException("Unexpected exception", e); + } + } + + @Override + public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult options) { + BazelDepGraphValue depGraphValue; + BazelModuleInspectorValue moduleInspector; + + SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor(); + LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class); + + EvaluationContext evaluationContext = + EvaluationContext.newBuilder() + .setNumThreads(threadsOption.threads) + .setEventHandler(env.getReporter()) + .build(); + + try { + env.syncPackageLoading(options); + + EvaluationResult evaluationResult = + skyframeExecutor.prepareAndGet( + ImmutableSet.of(BazelDepGraphValue.KEY, BazelModuleInspectorValue.KEY), + evaluationContext); + + if (evaluationResult.hasError()) { + Exception e = evaluationResult.getError().getException(); + String message = "Unexpected error during repository rule evaluation."; + if (e != null) { + message = e.getMessage(); + } + return reportAndCreateFailureResult(env, message, Code.INVALID_ARGUMENTS); + } + + depGraphValue = (BazelDepGraphValue) evaluationResult.get(BazelDepGraphValue.KEY); + + moduleInspector = + (BazelModuleInspectorValue) evaluationResult.get(BazelModuleInspectorValue.KEY); + + } catch (InterruptedException e) { + String errorMessage = "mod command interrupted: " + e.getMessage(); + env.getReporter().handle(Event.error(errorMessage)); + return BlazeCommandResult.detailedExitCode( + InterruptedFailureDetails.detailedExitCode(errorMessage)); + } catch (AbruptExitException e) { + env.getReporter().handle(Event.error(null, "Unknown error: " + e.getMessage())); + return BlazeCommandResult.detailedExitCode(e.getDetailedExitCode()); + } + + ModOptions modOptions = options.getOptions(ModOptions.class); + Preconditions.checkArgument(modOptions != null); + + if (options.getResidue().isEmpty()) { + String errorMessage = + String.format( + "No subcommand specified, choose one of : %s.", ModSubcommand.printValues()); + return reportAndCreateFailureResult(env, errorMessage, Code.MOD_COMMAND_UNKNOWN); + } + + // The first element in the residue must be the subcommand, and then comes a list of arguments. + String subcommandStr = options.getResidue().get(0); + ModSubcommand subcommand; + try { + subcommand = new ModSubcommandConverter().convert(subcommandStr); + } catch (OptionsParsingException e) { + String errorMessage = + String.format("Invalid subcommand, choose one from : %s.", ModSubcommand.printValues()); + return reportAndCreateFailureResult(env, errorMessage, Code.MOD_COMMAND_UNKNOWN); + } + List args = options.getResidue().subList(1, options.getResidue().size()); + + // Extract and check the --base_module argument first to use it when parsing the other args. + // Can only be a TargetModule or a repoName relative to the ROOT. + ModuleKey baseModuleKey; + AugmentedModule rootModule = moduleInspector.getDepGraph().get(ModuleKey.ROOT); + try { + ImmutableSet keys = + modOptions.baseModule.resolveToModuleKeys( + moduleInspector.getModulesIndex(), + moduleInspector.getDepGraph(), + rootModule.getDeps(), + rootModule.getUnusedDeps(), + false, + false); + if (keys.size() > 1) { + throw new InvalidArgumentException( + String.format( + "The --base_module option can only specify exactly one module version, choose one" + + " of: %s.", + keys.stream().map(ModuleKey::toString).collect(joining(", "))), + Code.INVALID_ARGUMENTS); + } + baseModuleKey = Iterables.getOnlyElement(keys); + } catch (InvalidArgumentException e) { + return reportAndCreateFailureResult( + env, + String.format( + "In --base_module %s option: %s (Note that unused modules cannot be used here)", + modOptions.baseModule, e.getMessage()), + Code.INVALID_ARGUMENTS); + } + + // The args can have different types depending on the subcommand, so create multiple containers + // which can be filled accordingly. + ImmutableSet argsAsModules = null; + ImmutableSortedSet argsAsExtensions = null; + ImmutableMap argsAsRepos = null; + + AugmentedModule baseModule = + Objects.requireNonNull(moduleInspector.getDepGraph().get(baseModuleKey)); + RepositoryMapping baseModuleMapping = depGraphValue.getFullRepoMapping(baseModuleKey); + try { + switch (subcommand) { + case GRAPH: + // GRAPH doesn't take extra arguments. + if (!args.isEmpty()) { + throw new InvalidArgumentException( + "the 'graph' command doesn't take extra arguments", Code.TOO_MANY_ARGUMENTS); + } + break; + case SHOW_REPO: + ImmutableMap.Builder targetToRepoName = + new ImmutableMap.Builder<>(); + for (String arg : args) { + try { + targetToRepoName.putAll( + ModuleArgConverter.INSTANCE + .convert(arg) + .resolveToRepoNames( + moduleInspector.getModulesIndex(), + moduleInspector.getDepGraph(), + baseModuleMapping)); + } catch (InvalidArgumentException | OptionsParsingException e) { + throw new InvalidArgumentException( + String.format( + "In repo argument %s: %s (Note that unused modules cannot be used here)", + arg, e.getMessage()), + Code.INVALID_ARGUMENTS, + e); + } + } + argsAsRepos = targetToRepoName.buildKeepingLast(); + break; + case SHOW_EXTENSION: + ImmutableSortedSet.Builder extensionsBuilder = + new ImmutableSortedSet.Builder<>(ModuleExtensionId.LEXICOGRAPHIC_COMPARATOR); + for (String arg : args) { + try { + extensionsBuilder.add( + ExtensionArgConverter.INSTANCE + .convert(arg) + .resolveToExtensionId( + moduleInspector.getModulesIndex(), + moduleInspector.getDepGraph(), + baseModule.getDeps(), + baseModule.getUnusedDeps())); + } catch (InvalidArgumentException | OptionsParsingException e) { + throw new InvalidArgumentException( + String.format("In extension argument: %s %s", arg, e.getMessage()), + Code.INVALID_ARGUMENTS, + e); + } + } + argsAsExtensions = extensionsBuilder.build(); + break; + default: + ImmutableSet.Builder keysBuilder = new ImmutableSet.Builder<>(); + for (String arg : args) { + try { + keysBuilder.addAll( + ModuleArgConverter.INSTANCE + .convert(arg) + .resolveToModuleKeys( + moduleInspector.getModulesIndex(), + moduleInspector.getDepGraph(), + baseModule.getDeps(), + baseModule.getUnusedDeps(), + modOptions.includeUnused, + /* warnUnused= */ true)); + } catch (InvalidArgumentException | OptionsParsingException e) { + throw new InvalidArgumentException( + String.format("In module argument %s: %s", arg, e.getMessage()), + Code.INVALID_ARGUMENTS); + } + } + argsAsModules = keysBuilder.build(); + } + } catch (InvalidArgumentException e) { + return reportAndCreateFailureResult(env, e.getMessage(), e.getCode()); + } + /* Extract and check the --from and --extension_usages argument */ + ImmutableSet fromKeys; + ImmutableSet usageKeys; + try { + fromKeys = + moduleArgListToKeys( + modOptions.modulesFrom, + moduleInspector.getModulesIndex(), + moduleInspector.getDepGraph(), + baseModule.getDeps(), + baseModule.getUnusedDeps(), + modOptions.includeUnused); + } catch (InvalidArgumentException e) { + return reportAndCreateFailureResult( + env, + String.format("In --from %s option: %s", modOptions.modulesFrom, e.getMessage()), + Code.INVALID_ARGUMENTS); + } + + try { + usageKeys = + moduleArgListToKeys( + modOptions.extensionUsages, + moduleInspector.getModulesIndex(), + moduleInspector.getDepGraph(), + baseModule.getDeps(), + baseModule.getUnusedDeps(), + modOptions.includeUnused); + } catch (InvalidArgumentException e) { + return reportAndCreateFailureResult( + env, + String.format( + "In --extension_usages %s option: %s (Note that unused modules cannot be used" + + " here)", + modOptions.extensionUsages, e.getMessage()), + Code.INVALID_ARGUMENTS); + } + + /* Extract and check the --extension_filter argument */ + Optional> filterExtensions = Optional.empty(); + if (subcommand.isGraph() && modOptions.extensionFilter != null) { + if (modOptions.extensionFilter.isEmpty()) { + filterExtensions = Optional.of(MaybeCompleteSet.completeSet()); + } else { + try { + filterExtensions = + Optional.of( + MaybeCompleteSet.copyOf( + extensionArgListToIds( + modOptions.extensionFilter, + moduleInspector.getModulesIndex(), + moduleInspector.getDepGraph(), + baseModule.getDeps(), + baseModule.getUnusedDeps()))); + } catch (InvalidArgumentException e) { + return reportAndCreateFailureResult( + env, + String.format( + "In --extension_filter %s option: %s", + modOptions.extensionFilter, e.getMessage()), + Code.INVALID_ARGUMENTS); + } + } + } + + ImmutableMap targetRepoRuleValues = null; + try { + // If subcommand is a SHOW, also request the BzlmodRepoRuleValues from Skyframe. + if (subcommand == ModSubcommand.SHOW_REPO) { + ImmutableSet skyKeys = + argsAsRepos.values().stream().map(BzlmodRepoRuleValue::key).collect(toImmutableSet()); + EvaluationResult result = + env.getSkyframeExecutor().prepareAndGet(skyKeys, evaluationContext); + if (result.hasError()) { + Exception e = result.getError().getException(); + String message = "Unexpected error during repository rule evaluation."; + if (e != null) { + message = e.getMessage(); + } + return reportAndCreateFailureResult(env, message, Code.INVALID_ARGUMENTS); + } + targetRepoRuleValues = + argsAsRepos.entrySet().stream() + .collect( + toImmutableMap( + Entry::getKey, + e -> + (BzlmodRepoRuleValue) + result.get(BzlmodRepoRuleValue.key(e.getValue())))); + } + } catch (InterruptedException e) { + String errorMessage = "mod command interrupted: " + e.getMessage(); + env.getReporter().handle(Event.error(errorMessage)); + return BlazeCommandResult.detailedExitCode( + InterruptedFailureDetails.detailedExitCode(errorMessage)); + } + + // Workaround to allow different default value for DEPS and EXPLAIN, and also use + // Integer.MAX_VALUE instead of the exact number string. + if (modOptions.depth < 1) { + switch (subcommand) { + case EXPLAIN: + modOptions.depth = 1; + break; + case DEPS: + modOptions.depth = 2; + break; + default: + modOptions.depth = Integer.MAX_VALUE; + } + } + + ModExecutor modExecutor = + new ModExecutor( + moduleInspector.getDepGraph(), + depGraphValue.getExtensionUsagesTable(), + moduleInspector.getExtensionToRepoInternalNames(), + filterExtensions, + modOptions, + new OutputStreamWriter( + env.getReporter().getOutErr().getOutputStream(), + modOptions.charset == UTF8 ? UTF_8 : US_ASCII)); + + switch (subcommand) { + case GRAPH: + modExecutor.graph(fromKeys); + break; + case DEPS: + modExecutor.graph(argsAsModules); + break; + case PATH: + modExecutor.path(fromKeys, argsAsModules); + break; + case ALL_PATHS: + case EXPLAIN: + modExecutor.allPaths(fromKeys, argsAsModules); + break; + case SHOW_REPO: + modExecutor.showRepo(targetRepoRuleValues); + break; + case SHOW_EXTENSION: + modExecutor.showExtension(argsAsExtensions, usageKeys); + break; + } + + return BlazeCommandResult.success(); + } + + /** Collects a list of {@link ModuleArg} into a set of {@link ModuleKey}s. */ + private static ImmutableSet moduleArgListToKeys( + ImmutableList argList, + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + ImmutableBiMap baseModuleDeps, + ImmutableBiMap baseModuleUnusedDeps, + boolean includeUnused) + throws InvalidArgumentException { + ImmutableSet.Builder allTargetKeys = new ImmutableSet.Builder<>(); + for (ModuleArg moduleArg : argList) { + allTargetKeys.addAll( + moduleArg.resolveToModuleKeys( + modulesIndex, depGraph, baseModuleDeps, baseModuleUnusedDeps, includeUnused, true)); + } + return allTargetKeys.build(); + } + + private static ImmutableSortedSet extensionArgListToIds( + ImmutableList args, + ImmutableMap> modulesIndex, + ImmutableMap depGraph, + ImmutableBiMap baseModuleDeps, + ImmutableBiMap baseModuleUnusedDeps) + throws InvalidArgumentException { + ImmutableSortedSet.Builder extensionsBuilder = + new ImmutableSortedSet.Builder<>(ModuleExtensionId.LEXICOGRAPHIC_COMPARATOR); + for (ExtensionArg arg : args) { + extensionsBuilder.add( + arg.resolveToExtensionId(modulesIndex, depGraph, baseModuleDeps, baseModuleUnusedDeps)); + } + return extensionsBuilder.build(); + } + + private static BlazeCommandResult reportAndCreateFailureResult( + CommandEnvironment env, String message, Code detailedCode) { + if (message.charAt(message.length() - 1) != '.') { + message = message.concat("."); + } + String fullMessage = + String.format( + message.concat(" Type '%s help mod' for syntax and help."), + env.getRuntime().getProductName()); + env.getReporter().handle(Event.error(fullMessage)); + return createFailureResult(fullMessage, detailedCode); + } + + private static BlazeCommandResult createFailureResult(String message, Code detailedCode) { + return BlazeCommandResult.detailedExitCode( + DetailedExitCode.of( + FailureDetail.newBuilder() + .setModCommand(FailureDetails.ModCommand.newBuilder().setCode(detailedCode).build()) + .setMessage(message) + .build())); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryCommand.java deleted file mode 100644 index 9495652ec440ea..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryCommand.java +++ /dev/null @@ -1,396 +0,0 @@ -// Copyright 2022 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.lib.bazel.commands; - -import static com.google.common.collect.ImmutableMap.toImmutableMap; -import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static java.nio.charset.StandardCharsets.US_ASCII; -import static java.nio.charset.StandardCharsets.UTF_8; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableBiMap; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue; -import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; -import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleValue; -import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; -import com.google.devtools.build.lib.bazel.bzlmod.Version; -import com.google.devtools.build.lib.bazel.commands.ModqueryOptions.Charset; -import com.google.devtools.build.lib.bazel.commands.ModqueryOptions.QueryType; -import com.google.devtools.build.lib.bazel.commands.ModqueryOptions.QueryTypeConverter; -import com.google.devtools.build.lib.bazel.commands.ModqueryOptions.TargetModule; -import com.google.devtools.build.lib.bazel.commands.ModqueryOptions.TargetModuleListConverter; -import com.google.devtools.build.lib.events.Event; -import com.google.devtools.build.lib.pkgcache.PackageOptions; -import com.google.devtools.build.lib.runtime.BlazeCommand; -import com.google.devtools.build.lib.runtime.BlazeCommandResult; -import com.google.devtools.build.lib.runtime.Command; -import com.google.devtools.build.lib.runtime.CommandEnvironment; -import com.google.devtools.build.lib.runtime.KeepGoingOption; -import com.google.devtools.build.lib.runtime.LoadingPhaseThreadsOption; -import com.google.devtools.build.lib.server.FailureDetails; -import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; -import com.google.devtools.build.lib.server.FailureDetails.ModqueryCommand.Code; -import com.google.devtools.build.lib.skyframe.SkyframeExecutor; -import com.google.devtools.build.lib.util.AbruptExitException; -import com.google.devtools.build.lib.util.DetailedExitCode; -import com.google.devtools.build.lib.util.InterruptedFailureDetails; -import com.google.devtools.build.skyframe.EvaluationContext; -import com.google.devtools.build.skyframe.EvaluationResult; -import com.google.devtools.build.skyframe.SkyKey; -import com.google.devtools.build.skyframe.SkyValue; -import com.google.devtools.common.options.OptionsParsingException; -import com.google.devtools.common.options.OptionsParsingResult; -import java.io.OutputStreamWriter; -import java.util.List; -import java.util.Objects; - -/** Queries the Bzlmod external dependency graph. */ -@Command( - name = ModqueryCommand.NAME, - // TODO(andreisolo): figure out which extra options are really needed - options = { - ModqueryOptions.class, - // Don't know what these do but were used in fetch - PackageOptions.class, - KeepGoingOption.class, - LoadingPhaseThreadsOption.class - }, - help = "resource:modquery.txt", - shortDescription = "Queries the Bzlmod external dependency graph", - allowResidue = true) -public final class ModqueryCommand implements BlazeCommand { - - public static final String NAME = "modquery"; - - @Override - public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult options) { - BazelModuleInspectorValue moduleInspector; - - SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor(); - LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class); - - EvaluationContext evaluationContext = - EvaluationContext.newBuilder() - .setNumThreads(threadsOption.threads) - .setEventHandler(env.getReporter()) - .build(); - - try { - env.syncPackageLoading(options); - - EvaluationResult evaluationResult = - skyframeExecutor.prepareAndGet( - ImmutableSet.of(BazelModuleInspectorValue.KEY), evaluationContext); - - if (evaluationResult.hasError()) { - Exception e = evaluationResult.getError().getException(); - String message = "Unexpected error during repository rule evaluation."; - if (e != null) { - message = e.getMessage(); - } - return reportAndCreateFailureResult(env, message, Code.INVALID_ARGUMENTS); - } - - moduleInspector = - (BazelModuleInspectorValue) evaluationResult.get(BazelModuleInspectorValue.KEY); - - } catch (InterruptedException e) { - String errorMessage = "Modquery interrupted: " + e.getMessage(); - env.getReporter().handle(Event.error(errorMessage)); - return BlazeCommandResult.detailedExitCode( - InterruptedFailureDetails.detailedExitCode(errorMessage)); - } catch (AbruptExitException e) { - env.getReporter().handle(Event.error(null, "Unknown error: " + e.getMessage())); - return BlazeCommandResult.detailedExitCode(e.getDetailedExitCode()); - } - - ModqueryOptions modqueryOptions = options.getOptions(ModqueryOptions.class); - Preconditions.checkArgument(modqueryOptions != null); - - if (options.getResidue().isEmpty()) { - String errorMessage = - String.format("No query type specified, choose one from : %s.", QueryType.printValues()); - return reportAndCreateFailureResult(env, errorMessage, Code.MODQUERY_COMMAND_UNKNOWN); - } - - // The first keyword in the residue must be the QueryType, and then comes a list of "arguments". - String queryInput = options.getResidue().get(0); - QueryType query; - try { - query = new QueryTypeConverter().convert(queryInput); - } catch (OptionsParsingException e) { - String errorMessage = - String.format("Invalid query type, choose one from : %s.", QueryType.printValues()); - return reportAndCreateFailureResult(env, errorMessage, Code.MODQUERY_COMMAND_UNKNOWN); - } - List args = options.getResidue().subList(1, options.getResidue().size()); - - // Arguments are structured as a list of comma-separated target lists for generality, - // even though there can only be 0 or 1 args so far. - ImmutableList> argsKeysList; - AugmentedModule rootModule = moduleInspector.getDepGraph().get(ModuleKey.ROOT); - try { - argsKeysList = - parseTargetArgs( - query.getArgNumber(), - moduleInspector.getModulesIndex(), - args, - rootModule.getDeps(), - rootModule.getUnusedDeps(), - modqueryOptions.includeUnused); - } catch (InvalidArgumentException e) { - return reportAndCreateFailureResult(env, e.getMessage(), e.getCode()); - } catch (OptionsParsingException e) { - return reportAndCreateFailureResult(env, e.getMessage(), Code.INVALID_ARGUMENTS); - } - /* Extract and check the --from argument */ - ImmutableSet fromKeys; - if (modqueryOptions.modulesFrom == null) { - fromKeys = ImmutableSet.of(ModuleKey.ROOT); - } else { - try { - fromKeys = - targetListToModuleKeySet( - modqueryOptions.modulesFrom, - moduleInspector.getModulesIndex(), - rootModule.getDeps(), - rootModule.getUnusedDeps(), - modqueryOptions.includeUnused); - } catch (InvalidArgumentException e) { - return reportAndCreateFailureResult(env, e.getMessage(), e.getCode()); - } - } - - ImmutableMap repoRuleValues = null; - // If the query is a SHOW, also request the BzlmodRepoRuleValues from SkyFrame. - // Unused modules do not have a BzlmodRepoRuleValue, so they are filtered out. - if (query == QueryType.SHOW) { - try { - ImmutableSet keys = - argsKeysList.get(0).stream() - .filter( - k -> - ModqueryExecutor.filterUnused( - k, modqueryOptions.includeUnused, false, moduleInspector.getDepGraph())) - .collect(toImmutableSet()); - ImmutableSet skyKeys = - keys.stream() - .map(k -> BzlmodRepoRuleValue.key(k.getCanonicalRepoName())) - .collect(toImmutableSet()); - EvaluationResult result = - env.getSkyframeExecutor().prepareAndGet(skyKeys, evaluationContext); - repoRuleValues = - keys.stream() - .collect( - toImmutableMap( - k -> k, - k -> - (BzlmodRepoRuleValue) - result.get(BzlmodRepoRuleValue.key(k.getCanonicalRepoName())))); - } catch (InterruptedException e) { - String errorMessage = "Modquery interrupted: " + e.getMessage(); - env.getReporter().handle(Event.error(errorMessage)); - return BlazeCommandResult.detailedExitCode( - InterruptedFailureDetails.detailedExitCode(errorMessage)); - } - } - - // Workaround to allow different default value for DEPS and EXPLAIN, and also use - // Integer.MAX_VALUE instead of the exact number string. - if (modqueryOptions.depth < 1) { - if (query == QueryType.EXPLAIN || query == QueryType.DEPS) { - modqueryOptions.depth = 1; - } else { - modqueryOptions.depth = Integer.MAX_VALUE; - } - } - - ModqueryExecutor modqueryExecutor = - new ModqueryExecutor( - moduleInspector.getDepGraph(), - modqueryOptions, - new OutputStreamWriter( - env.getReporter().getOutErr().getOutputStream(), - modqueryOptions.charset == Charset.UTF8 ? UTF_8 : US_ASCII)); - - switch (query) { - case TREE: - modqueryExecutor.tree(fromKeys); - break; - case DEPS: - modqueryExecutor.tree(argsKeysList.get(0)); - break; - case PATH: - modqueryExecutor.path(fromKeys, argsKeysList.get(0)); - break; - case ALL_PATHS: - modqueryExecutor.allPaths(fromKeys, argsKeysList.get(0)); - break; - case EXPLAIN: - modqueryExecutor.allPaths(fromKeys, argsKeysList.get(0)); - break; - case SHOW: - Preconditions.checkArgument(repoRuleValues != null); - modqueryExecutor.show(repoRuleValues); - break; - } - - return BlazeCommandResult.success(); - } - - /** - * A general parser for an undefined number of arguments. The arguments are comma-separated lists - * of {@link TargetModule}s. Each target {@link TargetModule} can represent a {@code repo_name}, - * as defined in the {@code MODULE.bazel} file of the root project, a specific version of a - * module, or all present versions of a module. The root module can only be specified by the - * {@code root} keyword, which takes precedence over the other above (in case of modules named - * root). - */ - @VisibleForTesting - public static ImmutableList> parseTargetArgs( - int requiredArgNum, - ImmutableMap> modulesIndex, - List args, - ImmutableBiMap rootDeps, - ImmutableBiMap rootUnusedDeps, - boolean includeUnused) - throws OptionsParsingException, InvalidArgumentException { - if (requiredArgNum != args.size()) { - throw new InvalidArgumentException( - String.format( - "Invalid number of arguments (provided %d, required %d).", - args.size(), requiredArgNum), - requiredArgNum > args.size() ? Code.MISSING_ARGUMENTS : Code.TOO_MANY_ARGUMENTS); - } - - TargetModuleListConverter converter = new TargetModuleListConverter(); - ImmutableList.Builder> argsKeysListBuilder = - new ImmutableList.Builder<>(); - - for (String arg : args) { - ImmutableList targetList = converter.convert(arg); - ImmutableSet argModuleKeys = - targetListToModuleKeySet( - targetList, modulesIndex, rootDeps, rootUnusedDeps, includeUnused); - argsKeysListBuilder.add(argModuleKeys); - } - return argsKeysListBuilder.build(); - } - - /** Collects a list of {@link TargetModule} into a set of {@link ModuleKey}s. */ - private static ImmutableSet targetListToModuleKeySet( - ImmutableList targetList, - ImmutableMap> modulesIndex, - ImmutableBiMap rootDeps, - ImmutableBiMap rootUnusedDeps, - boolean includeUnused) - throws InvalidArgumentException { - ImmutableSet.Builder allTargetKeys = new ImmutableSet.Builder<>(); - for (TargetModule targetModule : targetList) { - allTargetKeys.addAll( - targetToModuleKeySet( - targetModule, modulesIndex, rootDeps, rootUnusedDeps, includeUnused)); - } - return allTargetKeys.build(); - } - - /** - * Helper to check the module (and version) of the given {@link TargetModule} exists and retrieve - * it, (or retrieve all present versions if it's semantic specifies it, i.e. when - * Version == null). - */ - private static ImmutableSet targetToModuleKeySet( - TargetModule target, - ImmutableMap> modulesIndex, - ImmutableBiMap rootDeps, - ImmutableBiMap rootUnusedDeps, - boolean includeUnused) - throws InvalidArgumentException { - if (target.getName().isEmpty() && Objects.equals(target.getVersion(), Version.EMPTY)) { - return ImmutableSet.of(ModuleKey.ROOT); - } - if (rootDeps.containsKey(target.getName())) { - if (includeUnused && rootUnusedDeps.containsKey(target.getName())) { - return ImmutableSet.of( - rootDeps.get(target.getName()), rootUnusedDeps.get(target.getName())); - } - return ImmutableSet.of(rootDeps.get(target.getName())); - } - ImmutableSet existingKeys = modulesIndex.get(target.getName()); - - if (existingKeys == null) { - throw new InvalidArgumentException( - String.format("Module %s does not exist in the dependency graph.", target.getName()), - Code.INVALID_ARGUMENTS); - } - - if (target.getVersion() == null) { - return existingKeys; - } - ModuleKey key = ModuleKey.create(target.getName(), target.getVersion()); - if (!existingKeys.contains(key)) { - throw new InvalidArgumentException( - String.format( - "Module version %s@%s does not exist, available versions: %s.", - target.getName(), key, existingKeys), - Code.INVALID_ARGUMENTS); - } - return ImmutableSet.of(key); - } - - private static BlazeCommandResult reportAndCreateFailureResult( - CommandEnvironment env, String message, Code detailedCode) { - if (message.charAt(message.length() - 1) != '.') { - message = message.concat("."); - } - String fullMessage = - String.format( - message.concat(" Type '%s help modquery' for syntax and help."), - env.getRuntime().getProductName()); - env.getReporter().handle(Event.error(fullMessage)); - return createFailureResult(fullMessage, detailedCode); - } - - private static BlazeCommandResult createFailureResult(String message, Code detailedCode) { - return BlazeCommandResult.detailedExitCode( - DetailedExitCode.of( - FailureDetail.newBuilder() - .setModqueryCommand( - FailureDetails.ModqueryCommand.newBuilder().setCode(detailedCode).build()) - .setMessage(message) - .build())); - } - - /** - * Exception thrown when a user-input argument is invalid (wrong number of arguments or the - * specified modules do not exist). - */ - @VisibleForTesting - public static class InvalidArgumentException extends Exception { - private final Code code; - - private InvalidArgumentException(String message, Code code) { - super(message); - this.code = code; - } - - public Code getCode() { - return code; - } - } -} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryExecutor.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryExecutor.java deleted file mode 100644 index 2fcb9fa4ffc239..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryExecutor.java +++ /dev/null @@ -1,435 +0,0 @@ -// Copyright 2022 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.bazel.commands; - -import static com.google.common.collect.ImmutableMap.toImmutableMap; -import static java.util.stream.Collectors.toCollection; - -import com.google.auto.value.AutoValue; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.ImmutableSortedMap; -import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; -import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleValue; -import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; -import com.google.devtools.build.lib.bazel.commands.ModqueryExecutor.ResultNode.IsExpanded; -import com.google.errorprone.annotations.CanIgnoreReturnValue; -import java.io.PrintWriter; -import java.io.Writer; -import java.util.ArrayDeque; -import java.util.Comparator; -import java.util.Deque; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import javax.annotation.Nullable; - -/** - * Executes inspection queries for {@link - * com.google.devtools.build.lib.bazel.commands.ModqueryCommand}. - */ -public class ModqueryExecutor { - - private final ImmutableMap depGraph; - private final ModqueryOptions options; - private final PrintWriter printer; - - public ModqueryExecutor( - ImmutableMap depGraph, ModqueryOptions options, Writer writer) { - this.depGraph = depGraph; - this.options = options; - this.printer = new PrintWriter(writer); - } - - public void tree(ImmutableSet targets) { - ImmutableMap result = expandAndPrune(targets, ImmutableSet.of(), false); - printer.println(result); - printer.println("OUTPUT NOT IMPLEMENTED YET"); - } - - public void path(ImmutableSet from, ImmutableSet to) { - ImmutableMap result = expandAndPrune(from, to, true); - printer.println(result); - printer.println("OUTPUT NOT IMPLEMENTED YET"); - } - - public void allPaths(ImmutableSet from, ImmutableSet to) { - ImmutableMap result = expandAndPrune(from, to, false); - printer.println(result); - printer.println("OUTPUT NOT IMPLEMENTED YET"); - } - - public void show(ImmutableMap repoRuleValues) { - printer.println("OUTPUT NOT IMPLEMENTED YET"); - } - - /** - * The core function which produces the {@link ResultNode} graph for all the graph-generating - * queries above. First, it expands the result graph starting from the {@code from} modules, up - * until the {@code to} target modules if they are specified. If {@code singlePath} is set, it - * will only contain a single path to one of the targets.
- * Then it calls {@link ResultGraphPruner#pruneByDepth()} to prune nodes after the specified - * {@code depth} (root is at depth 0). If the query specifies any {@code to} targets, even if they - * are below the specified depth, they will still be included in the graph using some indirect - * (dotted) edges. If {@code from} nodes other than the root are specified, they will be pinned - * (connected directly under the root - using indirect edges if necessary). - */ - @VisibleForTesting - ImmutableMap expandAndPrune( - ImmutableSet from, ImmutableSet to, boolean singlePath) { - final MaybeCompleteSet coloredPaths = colorReversePathsToRoot(to); - ImmutableMap.Builder resultBuilder = new ImmutableMap.Builder<>(); - - ImmutableSet rootDirectChildren = - depGraph.get(ModuleKey.ROOT).getAllDeps(options.includeUnused).keySet(); - ImmutableSet rootPinnedChildren = - getPinnedChildrenOfRootInTheResultGraph(rootDirectChildren, from); - ResultNode.Builder rootBuilder = ResultNode.builder(); - rootPinnedChildren.stream() - .filter(coloredPaths::contains) - .forEach( - moduleKey -> { - if (rootDirectChildren.contains(moduleKey)) { - rootBuilder.addChild(moduleKey, IsExpanded.TRUE); - } else { - rootBuilder.addIndirectChild(moduleKey, IsExpanded.TRUE); - } - }); - resultBuilder.put(ModuleKey.ROOT, rootBuilder.build()); - - Set seen = new HashSet<>(rootPinnedChildren); - Deque toVisit = new ArrayDeque<>(rootPinnedChildren); - seen.add(ModuleKey.ROOT); - - while (!toVisit.isEmpty()) { - ModuleKey key = toVisit.pop(); - AugmentedModule module = depGraph.get(key); - ResultNode.Builder nodeBuilder = ResultNode.builder(); - nodeBuilder.setTarget(to.contains(key)); - - ImmutableSet moduleDeps = module.getAllDeps(options.includeUnused).keySet(); - for (ModuleKey childKey : moduleDeps) { - if (!coloredPaths.contains(childKey)) { - continue; - } - if (to.contains(childKey)) { - nodeBuilder.setTargetParent(true); - } - if (seen.contains(childKey)) { - // Single paths should not contain cycles or unexpanded (duplicate) children - // TODO(andreisolo): Move the single path extraction to DFS otherwise it can produce a - // wrong answer in cycle edge-case A -> B -> C -> B with target D will not find ABD - // \__ D - if (!singlePath) { - nodeBuilder.addChild(childKey, IsExpanded.FALSE); - } - continue; - } - nodeBuilder.addChild(childKey, IsExpanded.TRUE); - seen.add(childKey); - toVisit.add(childKey); - if (singlePath) { - break; - } - } - - resultBuilder.put(key, nodeBuilder.build()); - } - return new ResultGraphPruner(!to.isEmpty(), resultBuilder.buildOrThrow()).pruneByDepth(); - } - - private class ResultGraphPruner { - - private final Map oldResult; - private final Map resultBuilder; - private final Set parentStack; - private final boolean withTargets; - - /** - * Prunes the result tree after the specified depth using DFS (because some nodes may still - * appear after the max depth).
- * - * @param withTargets If set, it means that the result tree contains paths to some specific - * targets. This will cause some branches to contain, after the specified depths, some - * targets or target parents. As any other nodes omitted, transitive edges (embedding - * multiple edges) will be stored as indirect. - * @param oldResult The unpruned result graph. - */ - ResultGraphPruner(boolean withTargets, Map oldResult) { - this.oldResult = oldResult; - this.resultBuilder = new HashMap<>(); - this.parentStack = new HashSet<>(); - this.withTargets = withTargets; - } - - private ImmutableMap pruneByDepth() { - ResultNode.Builder rootBuilder = ResultNode.builder(); - resultBuilder.put(ModuleKey.ROOT, rootBuilder); - - parentStack.add(ModuleKey.ROOT); - - for (ModuleKey childKey : oldResult.get(ModuleKey.ROOT).getChildren().keySet()) { - rootBuilder.addChild(childKey, IsExpanded.TRUE); - visitVisible(childKey, 1, ModuleKey.ROOT, IsExpanded.TRUE); - } - for (ModuleKey childKey : oldResult.get(ModuleKey.ROOT).getIndirectChildren().keySet()) { - rootBuilder.addIndirectChild(childKey, IsExpanded.TRUE); - visitVisible(childKey, 1, ModuleKey.ROOT, IsExpanded.TRUE); - } - - // Build everything at the end to allow children to add themselves to their parent's - // adjacency list. - return resultBuilder.entrySet().stream() - .collect(toImmutableMap(Entry::getKey, e -> e.getValue().build())); - } - - // Handles graph traversal within the specified depth. - private void visitVisible( - ModuleKey moduleKey, int depth, ModuleKey parentKey, IsExpanded expanded) { - parentStack.add(moduleKey); - ResultNode oldNode = oldResult.get(moduleKey); - ResultNode.Builder nodeBuilder = ResultNode.builder(); - - resultBuilder.put(moduleKey, nodeBuilder); - nodeBuilder.setTarget(oldNode.isTarget()); - if (depth > 1) { - resultBuilder.get(parentKey).addChild(moduleKey, expanded); - } - - if (expanded == IsExpanded.FALSE) { - parentStack.remove(moduleKey); - return; - } - for (Entry e : oldNode.getChildren().entrySet()) { - ModuleKey childKey = e.getKey(); - IsExpanded childExpanded = e.getValue(); - if (notCycle(childKey)) { - if (depth < options.depth) { - visitVisible(childKey, depth + 1, moduleKey, childExpanded); - } else if (withTargets) { - visitDetached(childKey, moduleKey, moduleKey, childExpanded); - } - } else if (options.cycles) { - nodeBuilder.addChild(childKey, IsExpanded.FALSE); - } - } - parentStack.remove(moduleKey); - } - - // Detached mode is only present in withTargets and handles adding targets and target parents - // living below the specified depth to the graph. - private void visitDetached( - ModuleKey moduleKey, - ModuleKey parentKey, - ModuleKey lastVisibleParentKey, - IsExpanded expanded) { - parentStack.add(moduleKey); - ResultNode oldNode = oldResult.get(moduleKey); - ResultNode.Builder nodeBuilder = ResultNode.builder(); - nodeBuilder.setTarget(oldNode.isTarget()); - - if (oldNode.isTarget() || oldNode.isTargetParent()) { - ResultNode.Builder parentBuilder = resultBuilder.get(lastVisibleParentKey); - if (lastVisibleParentKey.equals(parentKey)) { - parentBuilder.addChild(moduleKey, expanded); - } else { - parentBuilder.addIndirectChild(moduleKey, expanded); - } - resultBuilder.put(moduleKey, nodeBuilder); - lastVisibleParentKey = moduleKey; - } - - if (expanded == IsExpanded.FALSE) { - parentStack.remove(moduleKey); - return; - } - for (Entry e : oldNode.getChildren().entrySet()) { - ModuleKey childKey = e.getKey(); - IsExpanded childExpanded = e.getValue(); - if (notCycle(childKey)) { - visitDetached(childKey, moduleKey, lastVisibleParentKey, childExpanded); - } else if (options.cycles) { - nodeBuilder.addChild(childKey, IsExpanded.FALSE); - } - } - parentStack.remove(moduleKey); - } - - private boolean notCycle(ModuleKey key) { - return !parentStack.contains(key); - } - } - - /** - * Return a list of modules that will be the direct children of the root in the result graph - * (original root's direct dependencies along with the specified targets). - */ - private ImmutableSet getPinnedChildrenOfRootInTheResultGraph( - ImmutableSet rootDirectDeps, ImmutableSet fromTargets) { - Set targetKeys = - fromTargets.stream() - .filter(k -> filterUnused(k, options.includeUnused, true, depGraph)) - .collect(toCollection(HashSet::new)); - if (fromTargets.contains(ModuleKey.ROOT)) { - targetKeys.addAll(rootDirectDeps); - } - return ImmutableSet.copyOf(targetKeys); - } - - /** - * Color all reverse paths from the target modules to the root so only modules which are part of - * these paths will be included in the output graph during the breadth-first traversal. - */ - private MaybeCompleteSet colorReversePathsToRoot(ImmutableSet targets) { - if (targets.isEmpty()) { - return MaybeCompleteSet.completeSet(); - } - - Set seen = new HashSet<>(targets); - Deque toVisit = new ArrayDeque<>(targets); - - while (!toVisit.isEmpty()) { - ModuleKey key = toVisit.pop(); - AugmentedModule module = depGraph.get(key); - Set parents = new HashSet<>(module.getDependants()); - if (options.includeUnused) { - parents.addAll(module.getOriginalDependants()); - } - for (ModuleKey parent : parents) { - if (seen.contains(parent)) { - continue; - } - seen.add(parent); - toVisit.add(parent); - } - } - - return MaybeCompleteSet.copyOf(seen); - } - - // Helper to filter unused (and unloaded) specified target that cannot be explained and print a - // warning of that. - static boolean filterUnused( - ModuleKey key, - boolean includeUnused, - boolean allowNotLoaded, - ImmutableMap dependenciesGraph) { - AugmentedModule module = dependenciesGraph.get(key); - if (key.equals(ModuleKey.ROOT)) { - return false; - } - if (!module.isUsed() && !includeUnused) { - return false; - } - if (!module.isLoaded()) { - return allowNotLoaded; - } - return true; - } - - @AutoValue - abstract static class ResultNode { - - /** Whether the module is one of the targets in a paths query. */ - abstract boolean isTarget(); - - /** - * Whether the module is a parent of one of the targets in a paths query (can also be a target). - */ - abstract boolean isTargetParent(); - - enum IsExpanded { - TRUE, - FALSE - } - - /** List of direct children. True if the children will be expanded in this branch. */ - abstract ImmutableSortedMap getChildren(); - - /** List of indirect children. True if the children will be expanded in this branch. */ - abstract ImmutableSortedMap getIndirectChildren(); - - static ResultNode.Builder builder() { - return new AutoValue_ModqueryExecutor_ResultNode.Builder() - .setTarget(false) - .setTargetParent(false); - } - - @AutoValue.Builder - abstract static class Builder { - - abstract ResultNode.Builder setTargetParent(boolean value); - - private final ImmutableSortedMap.Builder childrenBuilder = - childrenBuilder(ModuleKey.LEXICOGRAPHIC_COMPARATOR); - private final ImmutableSortedMap.Builder indirectChildrenBuilder = - indirectChildrenBuilder(ModuleKey.LEXICOGRAPHIC_COMPARATOR); - - abstract ResultNode.Builder setTarget(boolean value); - - abstract ImmutableSortedMap.Builder childrenBuilder( - Comparator comparator); - - abstract ImmutableSortedMap.Builder indirectChildrenBuilder( - Comparator comparator); - - @CanIgnoreReturnValue - final Builder addChild(ModuleKey value, IsExpanded expanded) { - childrenBuilder.put(value, expanded); - return this; - } - - @CanIgnoreReturnValue - final Builder addIndirectChild(ModuleKey value, IsExpanded expanded) { - indirectChildrenBuilder.put(value, expanded); - return this; - } - - abstract ResultNode build(); - } - } - - /** - * A set that either contains some elements or is the complete set (has a special - * constructor).
- * A complete set is stored internally as {@code null}. However, passing null to {@link - * #copyOf(Set)} is not allowed. Use {@link #completeSet()} instead. - */ - @AutoValue - abstract static class MaybeCompleteSet { - @Nullable - protected abstract ImmutableSet internalSet(); - - boolean contains(T value) { - if (internalSet() == null) { - return true; - } - return internalSet().contains(value); - } - - static MaybeCompleteSet copyOf(Set nullableSet) { - Preconditions.checkArgument(nullableSet != null); - return new AutoValue_ModqueryExecutor_MaybeCompleteSet(ImmutableSet.copyOf(nullableSet)); - } - - static MaybeCompleteSet completeSet() { - return new AutoValue_ModqueryExecutor_MaybeCompleteSet<>(null); - } - } -} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryOptions.java deleted file mode 100644 index c12675bfd03c4f..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModqueryOptions.java +++ /dev/null @@ -1,275 +0,0 @@ -// Copyright 2022 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.lib.bazel.commands; - -import static java.util.Arrays.stream; -import static java.util.stream.Collectors.joining; - -import com.google.auto.value.AutoValue; -import com.google.common.base.Ascii; -import com.google.common.base.Preconditions; -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableList; -import com.google.devtools.build.lib.bazel.bzlmod.Version; -import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException; -import com.google.devtools.common.options.Converter; -import com.google.devtools.common.options.Converters.CommaSeparatedNonEmptyOptionListConverter; -import com.google.devtools.common.options.EnumConverter; -import com.google.devtools.common.options.Option; -import com.google.devtools.common.options.OptionDocumentationCategory; -import com.google.devtools.common.options.OptionEffectTag; -import com.google.devtools.common.options.OptionsBase; -import com.google.devtools.common.options.OptionsParsingException; -import java.util.List; -import javax.annotation.Nullable; - -/** Options for ModqueryCommand */ -public class ModqueryOptions extends OptionsBase { - - @Option( - name = "from", - defaultValue = "root", - converter = TargetModuleListConverter.class, - documentationCategory = OptionDocumentationCategory.MODQUERY, - effectTags = {OptionEffectTag.EXECUTION}, - help = - "The module(s) starting from which the dependency graph query will be displayed. Check" - + " each query’s description for the exact semantic. Defaults to root.\n") - public ImmutableList modulesFrom; - - @Option( - name = "extra", - defaultValue = "false", - documentationCategory = OptionDocumentationCategory.MODQUERY, - effectTags = {OptionEffectTag.EXECUTION}, - help = - "The queries will also display the reason why modules were resolved to their current" - + " version (if changed). Defaults to true only for the explain query.") - public boolean extra; - - @Option( - name = "include_unused", - defaultValue = "false", - documentationCategory = OptionDocumentationCategory.MODQUERY, - effectTags = {OptionEffectTag.EXECUTION}, - help = - "The queries will also take into account and display the unused modules, which are not" - + " present in the module resolution graph after selection (due to the" - + " Minimal-Version Selection or override rules). This can have different effects for" - + " each of the query types i.e. include new paths in the all_paths command, or extra" - + " dependants in the explain command.\n") - public boolean includeUnused; - - @Option( - name = "depth", - defaultValue = "-1", - documentationCategory = OptionDocumentationCategory.MODQUERY, - effectTags = {OptionEffectTag.EXECUTION}, - help = - "Maximum display depth of the dependency tree. A depth of 1 displays the direct" - + " dependencies, for example. For tree, path and all_paths it defaults to" - + " Integer.MAX_VALUE, while for deps and explain it defaults to 1 (only displays" - + " direct deps of the root besides the target leaves and their parents).\n") - public int depth; - - @Option( - name = "cycles", - defaultValue = "false", - documentationCategory = OptionDocumentationCategory.MODQUERY, - effectTags = {OptionEffectTag.EXECUTION}, - help = - "Points out dependency cycles inside the displayed tree, which are normally ignored by" - + " default.") - public boolean cycles; - - @Option( - name = "charset", - defaultValue = "utf8", - converter = CharsetConverter.class, - documentationCategory = OptionDocumentationCategory.MODQUERY, - effectTags = {OptionEffectTag.TERMINAL_OUTPUT}, - help = - "Chooses the character set to use for the tree. Only affects text output. Valid values" - + " are \"utf8\" or \"ascii\". Default is \"utf8\"") - public Charset charset; - - @Option( - name = "output", - defaultValue = "text", - converter = OutputFormatConverter.class, - documentationCategory = OptionDocumentationCategory.MODQUERY, - effectTags = {OptionEffectTag.TERMINAL_OUTPUT}, - help = - "The format in which the query results should be printed. Allowed values for query are: " - + "text, json, graph") - public OutputFormat outputFormat; - - enum QueryType { - DEPS(1), - TREE(0), - ALL_PATHS(1), - PATH(1), - EXPLAIN(1), - SHOW(1); - - /* Store the number of arguments that it accepts for easy pre-check */ - private final int argNumber; - - QueryType(int argNumber) { - this.argNumber = argNumber; - } - - @Override - public String toString() { - return Ascii.toLowerCase(this.name()); - } - - public int getArgNumber() { - return argNumber; - } - - public static String printValues() { - return "(" + stream(values()).map(QueryType::toString).collect(joining(", ")) + ")"; - } - } - - /** Converts a query type option string to a properly typed {@link QueryType} */ - public static class QueryTypeConverter extends EnumConverter { - public QueryTypeConverter() { - super(QueryType.class, "query type"); - } - } - - enum Charset { - UTF8, - ASCII - } - - /** Converts a charset option string to a properly typed {@link Charset} */ - public static class CharsetConverter extends EnumConverter { - public CharsetConverter() { - super(Charset.class, "output charset"); - } - } - - enum OutputFormat { - TEXT, - JSON, - GRAPH - } - - /** Converts an output format option string to a properly typed {@link OutputFormat} */ - public static class OutputFormatConverter extends EnumConverter { - public OutputFormatConverter() { - super(OutputFormat.class, "output format"); - } - } - - /** Argument of a modquery converted from the form name@version or name. */ - @AutoValue - abstract static class TargetModule { - static TargetModule create(String name, Version version) { - return new AutoValue_ModqueryOptions_TargetModule(name, version); - } - - abstract String getName(); - - /** - * If it is null, it represents any (one or multiple) present versions of the module in the dep - * graph, which is different from the empty version - */ - @Nullable - abstract Version getVersion(); - } - - /** Converts a module target argument string to a properly typed {@link TargetModule} */ - static class TargetModuleConverter extends Converter.Contextless { - - @Override - public TargetModule convert(String input) throws OptionsParsingException { - String errorMessage = String.format("Cannot parse the given module argument: %s.", input); - Preconditions.checkArgument(input != null); - // The keyword root takes priority if any module is named the same it can only be referenced - // using the full key. - if (Ascii.equalsIgnoreCase(input, "root")) { - return TargetModule.create("", Version.EMPTY); - } else { - List splits = Splitter.on('@').splitToList(input); - if (splits.isEmpty() || splits.get(0).isEmpty()) { - throw new OptionsParsingException(errorMessage); - } - - if (splits.size() == 2) { - if (splits.get(1).equals("_")) { - return TargetModule.create(splits.get(0), Version.EMPTY); - } - if (splits.get(1).isEmpty()) { - throw new OptionsParsingException(errorMessage); - } - try { - return TargetModule.create(splits.get(0), Version.parse(splits.get(1))); - } catch (ParseException e) { - throw new OptionsParsingException(errorMessage, e); - } - - } else if (splits.size() == 1) { - return TargetModule.create(splits.get(0), null); - } else { - throw new OptionsParsingException(errorMessage); - } - } - } - - @Override - public String getTypeDescription() { - return "root, @ or "; - } - } - - /** Converts a comma-separated module list argument (i.e. A@1.0,B@2.0) */ - public static class TargetModuleListConverter - extends Converter.Contextless> { - - @Override - public ImmutableList convert(String input) throws OptionsParsingException { - CommaSeparatedNonEmptyOptionListConverter listConverter = - new CommaSeparatedNonEmptyOptionListConverter(); - TargetModuleConverter targetModuleConverter = new TargetModuleConverter(); - ImmutableList targetStrings = - listConverter.convert(input, /*conversionContext=*/ null); - ImmutableList.Builder targetModules = new ImmutableList.Builder<>(); - for (String targetInput : targetStrings) { - targetModules.add(targetModuleConverter.convert(targetInput, /*conversionContext=*/ null)); - } - return targetModules.build(); - } - - @Override - public String getTypeDescription() { - return "a list of s separated by comma"; - } - } - - static ModqueryOptions getDefaultOptions() { - ModqueryOptions options = new ModqueryOptions(); - options.depth = Integer.MAX_VALUE; - options.cycles = false; - options.includeUnused = false; - options.extra = false; - options.modulesFrom = ImmutableList.of(TargetModule.create("", Version.EMPTY)); - options.charset = Charset.UTF8; - options.outputFormat = OutputFormat.TEXT; - return options; - } -} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/mod.txt b/src/main/java/com/google/devtools/build/lib/bazel/commands/mod.txt new file mode 100644 index 00000000000000..01d576972cf61d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/mod.txt @@ -0,0 +1,28 @@ + +Usage: %{product} %{command} [