diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 14cc3e69f..9d2dba308 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -20,17 +20,39 @@ jobs: - run: cargo build --locked - run: cargo test - uses: dtolnay/install-buck2@latest - - name: Build example project (Ubuntu, macOS) + - name: Build example projects (Ubuntu, macOS) + if: matrix.os == 'ubuntu' || matrix.os == 'macos' run: |- - cd example - reindeer --third-party-dir third-party buckify - buck2 init --git + pushd examples/01-intro + reindeer buckify buck2 run "//project:test" - if: matrix.os == 'ubuntu' || matrix.os == 'macos' - - name: Build example project (Windows) + popd + + pushd examples/02-hybrid + reindeer buckify + buck2 run "//project:test" + cargo run -p project + popd + + pushd examples/03-complex-fixups + reindeer buckify + buck2 run "//project:test" + popd + - name: Build example projects (Windows) + if: matrix.os == 'windows' run: |- - cd example - & reindeer --third-party-dir third-party buckify - & buck2 init --git + cd examples/01-intro + & reindeer buckify & buck2 run "//project:test" - if: matrix.os == 'windows' + cd ../.. + + cd examples/02-hybrid + & reindeer buckify + & buck2 run "//project:test" + & cargo run -p project + cd ../.. + + cd examples/03-complex-fixups + & reindeer buckify + & buck2 run "//project:test" + cd ../.. diff --git a/docs/MANUAL.md b/docs/MANUAL.md index d9a416aca..88270a2e6 100644 --- a/docs/MANUAL.md +++ b/docs/MANUAL.md @@ -30,16 +30,19 @@ environment matches this.) ## An Example -See the [example directory](../example/README.md) as a starting point. This has: +See the [first example directory](../examples/01-intro/) as a starting point. +This has: - Your first-party code in "project" (though of course it can be anywhere and everywhere), and -- A [third-party](../example/third-party) directory which is managed by Reindeer -- A [`setup.sh`](../example/setup.sh) script to get you bootstrapped +- A [third-party](../examples/01-intro/third-party) directory which is managed + by Reindeer +- A [`setup.sh`](../examples/01-intro/setup.sh) script to get you bootstrapped Running `setup.sh` will build Reindeer (using Cargo) and then use it to vendor the small number of third-party dependencies defined in -[`third-party/Cargo.toml`](../third-party/Cargo.toml) and generate build rules +[`third-party/Cargo.toml`](../examples/01-intro/third-party/Cargo.toml) and +generate build rules for them in `third-party/BUCK`. I recommend using this as a starting template for your own project, at least diff --git a/example/third-party/fixups/blake3/fixups.toml b/example/third-party/fixups/blake3/fixups.toml deleted file mode 100644 index 8d888e08d..000000000 --- a/example/third-party/fixups/blake3/fixups.toml +++ /dev/null @@ -1,7 +0,0 @@ -[[buildscript]] - -[buildscript.cxx_library] -name = "blake3_avx512" -srcs = ["c/blake3_avx512.c"] -compiler_flags = ["-mavx512f", "-mavx512vl"] -headers = ["c/*.h"] diff --git a/example/third-party/macros/rust_third_party.bzl b/example/third-party/macros/rust_third_party.bzl deleted file mode 100644 index 4c43d4ee5..000000000 --- a/example/third-party/macros/rust_third_party.bzl +++ /dev/null @@ -1,7 +0,0 @@ -def third_party_rust_cxx_library(name, **kwargs): - # @lint-ignore BUCKLINT - native.cxx_library(name = name, **kwargs) - -def third_party_rust_prebuilt_cxx_library(name, **kwargs): - # @lint-ignore BUCKLINT - native.prebuilt_cxx_library(name = name, **kwargs) diff --git a/examples/01-intro/.buckconfig b/examples/01-intro/.buckconfig new file mode 100644 index 000000000..829d0c9ce --- /dev/null +++ b/examples/01-intro/.buckconfig @@ -0,0 +1,24 @@ +[cells] + root = . + prelude = prelude + toolchains = toolchains + none = none + +[cell_aliases] + config = prelude + ovr_config = prelude + fbcode = none + fbsource = none + fbcode_macros = none + buck = none + +# Uses a copy of the prelude bundled with the buck2 binary. You can alternatively delete this +# section and vendor a copy of the prelude to the `prelude` directory of your project. +[external_cells] + prelude = bundled + +[parser] + target_platform_detector_spec = target:root//...->prelude//platforms:default + +[build] + execution_platforms = prelude//platforms:default diff --git a/example/toolchains/.buckconfig b/examples/01-intro/.buckroot similarity index 100% rename from example/toolchains/.buckconfig rename to examples/01-intro/.buckroot diff --git a/examples/01-intro/.gitignore b/examples/01-intro/.gitignore new file mode 100644 index 000000000..c5ffced9a --- /dev/null +++ b/examples/01-intro/.gitignore @@ -0,0 +1,2 @@ +/buck-out +/target diff --git a/examples/01-intro/BUCK b/examples/01-intro/BUCK new file mode 100644 index 000000000..a6f2a23ca --- /dev/null +++ b/examples/01-intro/BUCK @@ -0,0 +1,7 @@ +# A list of available rules and their signatures can be found here: https://buck2.build/docs/prelude/globals/ + +genrule( + name = "hello_world", + out = "out.txt", + cmd = "echo BUILT BY BUCK2> $OUT", +) diff --git a/example/README.md b/examples/01-intro/README.md similarity index 61% rename from example/README.md rename to examples/01-intro/README.md index 0551595f0..814e9228f 100644 --- a/example/README.md +++ b/examples/01-intro/README.md @@ -8,42 +8,42 @@ generate a BUCK file of build rules for them. This will require a Rust installation (stable, but probably fairly recent), and Buck to actually make use of the generated files. -You can learn more about Buck at [buck.build](https://buck.build). The -[getting started](https://buck.build/setup/getting_started.html) page should +You can learn more about Buck at [buck.build](https://buck2.build). The +[getting started](https://buck2.build/docs/about/getting_started/) page should help with getting it installed. ## Buck and its Configuration The `.buckconfig` file both configures Buck and tells it where the top of the -"cell" is (current working tree). This only contains some very minimal Rust -configuration; most notable is overriding the default to Rust 2018. - -(`.buck-java11` won't generally be needed.) +"cell" is (current working tree). ## Reindeer configuration The files and directories Reindeer cares about are under `third-party/`: -- reindeer.toml - Reindeer's configuration. The directory containing this file +- `reindeer.toml` - Reindeer's configuration. The directory containing this + file is also the base for any relative paths mentioned in the file. -- Cargo.toml - You edit this to specify which packages you want to import, along +- `Cargo.toml` - You edit this to specify which packages you want to import, + along with other settings like features, patches and so on, using the full syntax Cargo allows -- Cargo.lock - The resolved dependencies -- BUCK - The generated Buck build rules (once generated) -- .gitignore - This is used to ignore various inconvenient files in vendored +- `Cargo.lock` - The resolved dependencies +- `BUCK` - The generated Buck build rules (once generated) +- `.gitignore` - This is used to ignore various inconvenient files in vendored code. Reindeer itself will look at this to edit them out of the vendored checksum.json files so that Cargo doesn't get upset. In addition to these files, there are a few significant directories: -- vendor/ - where all the vendored code goes -- fixups/ - fixups tell Reindeer how to handle packages with build scripts and +- `vendor/` - where all the vendored code goes +- `fixups/` - fixups tell Reindeer how to handle packages with build scripts + and other special cases; most packages won't need anything here -- macros/ - Buck macros which map from the rules Reindeer generates to the +- `macros/` - Buck macros which map from the rules Reindeer generates to the actual build environment. This directory is not hard-coded and could be anywhere. The macros included here are quite minimal. -- top/ - Cargo needs a dummy package for the Cargo.toml (it doesn't allow a +- `top/` - Cargo needs a dummy package for the Cargo.toml (it doesn't allow a package which is _all_ dependencies) ## Project diff --git a/examples/01-intro/project/BUCK b/examples/01-intro/project/BUCK new file mode 100644 index 000000000..de22a6f15 --- /dev/null +++ b/examples/01-intro/project/BUCK @@ -0,0 +1,7 @@ +rust_binary( + name = "test", + srcs = ["src/main.rs"], + deps = [ + "//third-party:once_cell", + ], +) diff --git a/examples/01-intro/project/src/main.rs b/examples/01-intro/project/src/main.rs new file mode 100644 index 000000000..810e10952 --- /dev/null +++ b/examples/01-intro/project/src/main.rs @@ -0,0 +1,16 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use once_cell::sync::Lazy; + +const MAGIC: &str = "this is a magic string"; + +static SPECIAL: Lazy = Lazy::new(|| MAGIC.to_string()); + +fn main() { + println!("static {}", &*SPECIAL); +} diff --git a/examples/01-intro/reindeer.toml b/examples/01-intro/reindeer.toml new file mode 100644 index 000000000..8158847c1 --- /dev/null +++ b/examples/01-intro/reindeer.toml @@ -0,0 +1,48 @@ +## +## Reindeer Config +## +## This file sets all of Reindeer's basic configuration. This file also marks +## the top of the reindeer-managed directory, as all other paths are relative to +## this one (both paths mentioned in this file, and implicit). +## +## Reindeer is under active development, and the layout and options in this file +## may change. + +# Write output to third-party/BUCK +# This also sets the default input (manifest_path) to third-party/Cargo.toml +third_party_dir = "third-party" + +# If a fixups.toml file is needed (eg, the package has a build.rs), then +# generate a template fixups.toml to be edited. +fixup_templates = true + +# Configuration for generated BUCK file +[buck] +# Name of the generated file +file_name = "BUCK" # default + +# Rules used for various kinds of targets. These rules don't directly +# correspond with BUCK rules - they have extra attributes such as +# per-platform settings. The intent is that you provide a set of macro +# definitions which resolve them to appropriate underlying rules +# suitable for your environment. (This may also work for Buck-like +# build systems such as Bazel.) +rust_library = "cargo.rust_library" # A plain Rust library +rust_binary = "cargo.rust_binary" # A Rust executable +buildscript_genrule = "buildscript_run" # Rule for running a build script to produce rustc args and generated sources + +# Load the macros to which the rules above will resolve. +buckfile_imports = """ +load("@prelude//rust:cargo_buildscript.bzl", "buildscript_run") +load("@prelude//rust:cargo_package.bzl", "cargo") +""" + +# Banner comment for the generated BUCK File. +generated_file_header = """ +## +## \u0040generated by reindeer +## Do not edit by hand. +## +## See README.md for directions on how to update this. +## +""" diff --git a/example/setup.sh b/examples/01-intro/setup.sh similarity index 80% rename from example/setup.sh rename to examples/01-intro/setup.sh index a83bd0caa..917803a76 100755 --- a/example/setup.sh +++ b/examples/01-intro/setup.sh @@ -4,11 +4,11 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -# Generate file 'example/third-party/BUCK'. +# Generate file 'third-party/BUCK'. set -e -(cd ..; cargo build) +(cd ../..; cargo build) # Build a BUCK file to build third-party crates. # @@ -19,4 +19,4 @@ set -e # typically commit these fixups and the generated third-party/BUCK in the same # commit as above. -../target/debug/reindeer --third-party-dir third-party buckify +../../target/debug/reindeer --third-party-dir third-party buckify diff --git a/example/third-party/.gitignore b/examples/01-intro/third-party/.gitignore similarity index 91% rename from example/third-party/.gitignore rename to examples/01-intro/third-party/.gitignore index c18f3dc43..cd2eec533 100644 --- a/example/third-party/.gitignore +++ b/examples/01-intro/third-party/.gitignore @@ -1,9 +1,6 @@ # Ignore Cargo-related stuff -.cargo/registry -.cargo/git -/registry -/git -.package-cache +.cargo +target # Various cruft in vendored packages vendor/*/target diff --git a/examples/01-intro/third-party/BUCK b/examples/01-intro/third-party/BUCK new file mode 100644 index 000000000..bea2e1f09 --- /dev/null +++ b/examples/01-intro/third-party/BUCK @@ -0,0 +1,38 @@ +## +## @generated by reindeer +## Do not edit by hand. +## +## See README.md for directions on how to update this. +## + +load("@prelude//rust:cargo_buildscript.bzl", "buildscript_run") +load("@prelude//rust:cargo_package.bzl", "cargo") + +alias( + name = "once_cell", + actual = ":once_cell-1.20.2", + visibility = ["PUBLIC"], +) + +http_archive( + name = "once_cell-1.20.2.crate", + sha256 = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775", + strip_prefix = "once_cell-1.20.2", + urls = ["https://static.crates.io/crates/once_cell/1.20.2/download"], + visibility = [], +) + +cargo.rust_library( + name = "once_cell-1.20.2", + srcs = [":once_cell-1.20.2.crate"], + crate = "once_cell", + crate_root = "once_cell-1.20.2.crate/src/lib.rs", + edition = "2021", + features = [ + "alloc", + "default", + "race", + "std", + ], + visibility = [], +) diff --git a/examples/01-intro/third-party/Cargo.lock b/examples/01-intro/third-party/Cargo.lock new file mode 100644 index 000000000..dbc1b80c6 --- /dev/null +++ b/examples/01-intro/third-party/Cargo.lock @@ -0,0 +1,16 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "rust-third-party" +version = "0.0.0" +dependencies = [ + "once_cell", +] diff --git a/example/third-party/Cargo.toml b/examples/01-intro/third-party/Cargo.toml similarity index 89% rename from example/third-party/Cargo.toml rename to examples/01-intro/third-party/Cargo.toml index 12ee25d62..8dea07814 100644 --- a/example/third-party/Cargo.toml +++ b/examples/01-intro/third-party/Cargo.toml @@ -15,7 +15,6 @@ path = "top/main.rs" # List of packages to be imported, with version constraints, features # and all options Cargo supports. [dependencies] -blake3 = { version = "0.1", features = ["c_avx512"] } once_cell = "1.4" # Local patches - typically git references diff --git a/examples/01-intro/third-party/top/main.rs b/examples/01-intro/third-party/top/main.rs new file mode 100644 index 000000000..f328e4d9d --- /dev/null +++ b/examples/01-intro/third-party/top/main.rs @@ -0,0 +1 @@ +fn main() {} diff --git a/examples/01-intro/toolchains/BUCK b/examples/01-intro/toolchains/BUCK new file mode 100644 index 000000000..b3ac5bc3c --- /dev/null +++ b/examples/01-intro/toolchains/BUCK @@ -0,0 +1,5 @@ +load("@prelude//toolchains:demo.bzl", "system_demo_toolchains") + +# All the default toolchains, suitable for a quick demo or early prototyping. +# Most real projects should copy/paste the implementation to configure them. +system_demo_toolchains() \ No newline at end of file diff --git a/example/.buckconfig b/examples/02-hybrid/.buckconfig similarity index 89% rename from example/.buckconfig rename to examples/02-hybrid/.buckconfig index 7d84382e8..126e8b381 100644 --- a/example/.buckconfig +++ b/examples/02-hybrid/.buckconfig @@ -10,6 +10,9 @@ fbcode = none fbsource = none buck = none +[external_cells] +prelude = bundled + [parser] target_platform_detector_spec = target:root//...->prelude//platforms:default diff --git a/examples/02-hybrid/.buckroot b/examples/02-hybrid/.buckroot new file mode 100644 index 000000000..e69de29bb diff --git a/example/.buckversion b/examples/02-hybrid/.buckversion similarity index 100% rename from example/.buckversion rename to examples/02-hybrid/.buckversion diff --git a/examples/02-hybrid/.gitignore b/examples/02-hybrid/.gitignore new file mode 100644 index 000000000..c5ffced9a --- /dev/null +++ b/examples/02-hybrid/.gitignore @@ -0,0 +1,2 @@ +/buck-out +/target diff --git a/examples/02-hybrid/Cargo.lock b/examples/02-hybrid/Cargo.lock new file mode 100644 index 000000000..bf344e218 --- /dev/null +++ b/examples/02-hybrid/Cargo.lock @@ -0,0 +1,16 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "project" +version = "0.0.0" +dependencies = [ + "once_cell", +] diff --git a/examples/02-hybrid/Cargo.toml b/examples/02-hybrid/Cargo.toml new file mode 100644 index 000000000..808e5f0ed --- /dev/null +++ b/examples/02-hybrid/Cargo.toml @@ -0,0 +1,7 @@ +[workspace] + +members = ["project"] +resolver = "2" + +# Local patches - typically git references +[patch.crates-io] diff --git a/examples/02-hybrid/README.md b/examples/02-hybrid/README.md new file mode 100644 index 000000000..23844f0ed --- /dev/null +++ b/examples/02-hybrid/README.md @@ -0,0 +1,40 @@ +# Hybrid Cargo + Buck project + +This example is for using both Cargo and Buck on the same project. This is +useful if you want to: + +- use Cargo to do things like `cargo fmt`; +- have it work out of the box with `rust-analyzer`, so it's easy to contribute; +- ultimately build your code with Buck; +- or support building with either. + +The key is the `reindeer.toml` in the root, which stitches together a Cargo +workspace and a Buck package (`third-party`) to buckify Cargo dependencies +into. + +## Running the example + +```sh +cargo install --path . +cd examples/02-hybrid + +# Note that both buck and cargo can build this project +buck2 run //project:test +cargo run -p project + +# Add a dependency if you like +cargo add -p project rand + +# Buckify deps. Do this every time you modify the Cargo.tomls, in general. +reindeer buckify + +# Modify project/BUCK to add `//third-party:rand` to the deps list. +# Buck should still build the project, and the rand crate should be available +# in main.rs. +``` + +## Missing from this example + +This example does no vendoring or anything fancy with fixups. You will have to +follow the other examples to get a non-trivial Cargo project built with Buck. + diff --git a/examples/02-hybrid/project/BUCK b/examples/02-hybrid/project/BUCK new file mode 100644 index 000000000..691f18fb0 --- /dev/null +++ b/examples/02-hybrid/project/BUCK @@ -0,0 +1,9 @@ +rust_binary( + name = "test", + srcs = ["src/main.rs"], + deps = [ + "//third-party:once_cell", + # see README.md + # "//third-party:rand", + ], +) diff --git a/examples/02-hybrid/project/Cargo.toml b/examples/02-hybrid/project/Cargo.toml new file mode 100644 index 000000000..a53a65bd3 --- /dev/null +++ b/examples/02-hybrid/project/Cargo.toml @@ -0,0 +1,11 @@ +[package] +# Pseudo-package whose dependencies are imported and buckified +name = "project" +version = "0.0.0" +edition = "2021" +publish = false + +# List of packages to be imported, with version constraints, features +# and all options Cargo supports. +[dependencies] +once_cell = "1.4" diff --git a/examples/02-hybrid/project/src/main.rs b/examples/02-hybrid/project/src/main.rs new file mode 100644 index 000000000..d94b35a98 --- /dev/null +++ b/examples/02-hybrid/project/src/main.rs @@ -0,0 +1,24 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use once_cell::sync::Lazy; + +const MAGIC: &str = "this is a magic string"; + +static SPECIAL: Lazy = Lazy::new(|| MAGIC.to_string()); + +fn main() { + println!("static {}", &*SPECIAL); + + // 1. Uncomment this line + // 2. cargo add -p project rand + // 3. reindeer buckify + // 4. modify the BUCK file to include "//third-party:rand" + // 5. buck2 run //project:test + // + // println!("random {}", rand::random::()); +} diff --git a/examples/02-hybrid/reindeer.toml b/examples/02-hybrid/reindeer.toml new file mode 100644 index 000000000..57ee7f008 --- /dev/null +++ b/examples/02-hybrid/reindeer.toml @@ -0,0 +1,31 @@ +## +## Reindeer Config for a hybrid Cargo + Buck project +## + +# Paths +# +# This example has a workspace Cargo.toml. For all default workspace members, +# it will recursively buckify their dependencies to third-party/BUCK. +manifest_path = "Cargo.toml" +third_party_dir = "third-party" +fixup_templates = true + +# Configuration for generated BUCK file +# See other examples +[buck] +file_name = "BUCK" +rust_library = "cargo.rust_library" +rust_binary = "cargo.rust_binary" +buildscript_genrule = "buildscript_run" +buckfile_imports = """ +load("@prelude//rust:cargo_buildscript.bzl", "buildscript_run") +load("@prelude//rust:cargo_package.bzl", "cargo") +""" +generated_file_header = """ +## +## \u0040generated by reindeer +## Do not edit by hand. +## +## See README.md for directions on how to update this. +## +""" diff --git a/examples/02-hybrid/setup.sh b/examples/02-hybrid/setup.sh new file mode 100755 index 000000000..400437c57 --- /dev/null +++ b/examples/02-hybrid/setup.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +# Generate file 'third-party/BUCK'. + +set -e + +(cd ../..; cargo build) + +# Build a BUCK file to build third-party crates. +# +# This will resolve all the dependencies, and create or update +# third-party/Cargo.lock as required. +# +# It will create a template fixup.toml which you can edit as needed. You would +# typically commit these fixups and the generated third-party/BUCK in the same +# commit as above. + +../../target/debug/reindeer buckify diff --git a/examples/02-hybrid/third-party/.gitignore b/examples/02-hybrid/third-party/.gitignore new file mode 100644 index 000000000..0bee95aae --- /dev/null +++ b/examples/02-hybrid/third-party/.gitignore @@ -0,0 +1,5 @@ +# Ignore Cargo +# If you need to adjust settings in .cargo/config.toml, put one in a parent directory. +.cargo +target + diff --git a/examples/02-hybrid/third-party/BUCK b/examples/02-hybrid/third-party/BUCK new file mode 100644 index 000000000..bea2e1f09 --- /dev/null +++ b/examples/02-hybrid/third-party/BUCK @@ -0,0 +1,38 @@ +## +## @generated by reindeer +## Do not edit by hand. +## +## See README.md for directions on how to update this. +## + +load("@prelude//rust:cargo_buildscript.bzl", "buildscript_run") +load("@prelude//rust:cargo_package.bzl", "cargo") + +alias( + name = "once_cell", + actual = ":once_cell-1.20.2", + visibility = ["PUBLIC"], +) + +http_archive( + name = "once_cell-1.20.2.crate", + sha256 = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775", + strip_prefix = "once_cell-1.20.2", + urls = ["https://static.crates.io/crates/once_cell/1.20.2/download"], + visibility = [], +) + +cargo.rust_library( + name = "once_cell-1.20.2", + srcs = [":once_cell-1.20.2.crate"], + crate = "once_cell", + crate_root = "once_cell-1.20.2.crate/src/lib.rs", + edition = "2021", + features = [ + "alloc", + "default", + "race", + "std", + ], + visibility = [], +) diff --git a/examples/02-hybrid/toolchains/.buckconfig b/examples/02-hybrid/toolchains/.buckconfig new file mode 100644 index 000000000..e69de29bb diff --git a/example/toolchains/BUCK b/examples/02-hybrid/toolchains/BUCK similarity index 100% rename from example/toolchains/BUCK rename to examples/02-hybrid/toolchains/BUCK diff --git a/examples/03-complex-fixups/.buckconfig b/examples/03-complex-fixups/.buckconfig new file mode 100644 index 000000000..126e8b381 --- /dev/null +++ b/examples/03-complex-fixups/.buckconfig @@ -0,0 +1,21 @@ +[repositories] +root = . +prelude = prelude +toolchains = toolchains +none = none + +[repository_aliases] +config = prelude +fbcode = none +fbsource = none +buck = none + +[external_cells] +prelude = bundled + +[parser] +target_platform_detector_spec = target:root//...->prelude//platforms:default + +[rust] +default_edition = 2021 +remap_src_paths = yes diff --git a/examples/03-complex-fixups/.buckroot b/examples/03-complex-fixups/.buckroot new file mode 100644 index 000000000..e69de29bb diff --git a/examples/03-complex-fixups/.buckversion b/examples/03-complex-fixups/.buckversion new file mode 100644 index 000000000..a0f9a4b4b --- /dev/null +++ b/examples/03-complex-fixups/.buckversion @@ -0,0 +1 @@ +latest diff --git a/examples/03-complex-fixups/.gitignore b/examples/03-complex-fixups/.gitignore new file mode 100644 index 000000000..0a0ddb2ef --- /dev/null +++ b/examples/03-complex-fixups/.gitignore @@ -0,0 +1 @@ +/buck-out diff --git a/examples/03-complex-fixups/PACKAGE b/examples/03-complex-fixups/PACKAGE new file mode 100644 index 000000000..4c7c54e57 --- /dev/null +++ b/examples/03-complex-fixups/PACKAGE @@ -0,0 +1,22 @@ +load("@prelude//cfg/modifier:set_cfg_modifiers.bzl", "set_cfg_modifiers") +load("@prelude//rust:with_workspace.bzl", "with_rust_workspace") +load("@root//config:set_cfg_constructor.bzl", "set_cfg_constructor") + +# This enables the Config Modifiers RFC. +# Described at +# https://github.com/facebook/buck2/blob/main/docs/rfcs/cfg-modifiers/api.md +# +# This function can only be called from the repository root `PACKAGE` file. +ALIASES = { + "debug": "root//config/mode:debug", + "release": "root//config/mode:release", +} +set_cfg_constructor(aliases = ALIASES) + +set_cfg_modifiers( + cfg_modifiers = [ + # default is debug mode. + # override with -m release + "root//config/mode:debug", + ], +) diff --git a/examples/03-complex-fixups/README.md b/examples/03-complex-fixups/README.md new file mode 100644 index 000000000..44000399f --- /dev/null +++ b/examples/03-complex-fixups/README.md @@ -0,0 +1,133 @@ +# Example Reindeer+Buck project + +## Getting Started + +The `setup.sh` script will build Reindeer, vendor some Cargo packages and +generate a BUCK file of build rules for them. + +This will require a Rust installation (stable, but probably fairly recent), and +Buck to actually make use of the generated files. + +## Vendoring + +This example enables vendoring of the source files for the third party +crates (see `reindeer.toml` for how). If you run `reindeer vendor` and then +`reindeer buckify`, you get all the crate sources under `third-party/vendor`. +The idea is to check all these files into source control. + +Note the file `third-party/.gitignore`. This is used to ignore various +inconvenient files in vendored code. Reindeer itself will look at this to +edit them out of the vendored checksum.json files so that Cargo doesn't +get upset. + +There is another option if vendoring would include so many files that it wrecks +your Git performance and you don't have the capacity to tune it: +`vendor = "local-registry"`. This saves tarballs of the crates, instead of +expanding them, and relies on having +[`cargo-local-registry`](https://github.com/dhovart/cargo-local-registry) +installed and Buck2 version `2025-01-15` or later. + +## Fixups + +This example has a few dependencies to illustrate the fixups system. + +### `libc` [toml](third-party/fixups/libc/) + +This one is easy. The `libc` crate has a build script, aka `build.rs`, which is +used to enable `--cfg` flags on certain targets. + +If you delete the `third-party/fixups/libc` directory, and then `reindeer buckify` again, you'll see a template `fixups.toml` generated, suggesting that +you add some directives to handle the build script. You can then replace the +whole TOML file with: + +```toml +[[buildscript]] +[buildscript.rustc_flags] +``` + +### `crossbeam-utils` [toml](third-party/fixups/crossbeam-utils/) + +This is an indirect dependency of `crossbeam-queue`. +It has a build script, which you can see +[here](https://github.com/crossbeam-rs/crossbeam/blob/17fb8417a83a2694b6f8a37198cd20b34b621baf/crossbeam-utils/build.rs). + +So we can add a directive to run that build script and have Buck add the flags +it emits to rustc when we compile the crate: + +```toml +[[buildscript]] +[buildscript.rustc_flags] +``` + +However, that build script has the following line: + +```rust + env!("CARGO_PKG_NAME"), +``` + +Which will fail without some modification. Buck does not provide such +environment variables by default. You can tell reindeer to add them: + +```toml +cargo_env = true + +[[buildscript]] +[buildscript.rustc_flags] +``` + +### `blake3` [toml](third-party/fixups/blake3/) + +This one is huge, basically replacing a long `build.rs` using the `cc` +crate to invoke a C compiler. The `cc` invocations are manually translated to +`[buildscript.cxx_library]` fixups, which reindeer turns into `cxx_library` targets +for Buck to build. It's more complex still, because the blake3 crate works on +many platforms, so there are many variations of the cxx_library to generate +with different source files and C flags. + +If you come across a crate like this, for example `zstd` or `winapi`, you should check +whether someone has already added fixups for it in the [Buck2 repository's fixups +folder](https://github.com/facebook/buck2/tree/main/shim/third-party/rust/fixups). + +## Extra BUCK file imports + +Reindeer is very customizable, but it can't do everything. This example also +demonstrates adding extra macro imports to the BUCK file, and wrapping the +prelude's rules with your own Starlark code. + +A sample use case for this is customizing the opt-levels of third party code by +modifying the `rustc_flags` attribute as it passes through. See the code in + +- `config/`, specifying the debug/release constraints +- `PACKAGE`, enabling config modifiers and setting up aliases like `buck2 build -m release` +- `third-party/macros/rust_third_party.bzl`, which defines macros that will be + imported by `third-party/BUCK`, and selects opt-level based on the + `//config/mode:debug / release` constraints +- `reindeer.toml`, which tells reindeer to generate a macro call for each crate to use our + custom macros. + +## Reindeer configuration + +`reindeer.toml` - Reindeer's configuration. The directory containing this +file is also the base for any relative paths mentioned in the file. + +The files and directories Reindeer cares about are under `third-party/`: + +- `Cargo.toml` - You edit this to specify which packages you want to import, + along + with other settings like features, patches and so on, using the full syntax + Cargo allows +- `Cargo.lock` - The resolved dependencies +- `BUCK` - The generated Buck build rules (once generated) +- `.gitignore` - This is used to ignore various inconvenient files in vendored + code. Reindeer itself will look at this to edit them out of the vendored + checksum.json files so that Cargo doesn't get upset. + +## Project + +The `project/` directory represents some end-user code. There's nothing notable +about it aside from its references to `//third-party:...` for its third-party +dependencies. + +Once everything is set up, you should be able to build it with +`buck build //project:test` to build the executable or `buck run //project:test` +to just build and run in situ. diff --git a/examples/03-complex-fixups/config/mode/BUCK b/examples/03-complex-fixups/config/mode/BUCK new file mode 100644 index 000000000..70f24d80b --- /dev/null +++ b/examples/03-complex-fixups/config/mode/BUCK @@ -0,0 +1,15 @@ +config_setting( + name = "debug", + constraint_values = [ + "//config/mode/constraints:debug", + ], + visibility = ["PUBLIC"], +) + +config_setting( + name = "release", + constraint_values = [ + "//config/mode/constraints:release", + ], + visibility = ["PUBLIC"], +) diff --git a/examples/03-complex-fixups/config/mode/constraints/BUCK b/examples/03-complex-fixups/config/mode/constraints/BUCK new file mode 100644 index 000000000..f09877e18 --- /dev/null +++ b/examples/03-complex-fixups/config/mode/constraints/BUCK @@ -0,0 +1,17 @@ +constraint_setting( + name = "mode", + visibility = ["PUBLIC"], +) + +constraint_value( + name = "debug", + constraint_setting = ":mode", + visibility = ["PUBLIC"], +) + +constraint_value( + name = "release", + constraint_setting = ":mode", + visibility = ["PUBLIC"], +) + diff --git a/examples/03-complex-fixups/config/set_cfg_constructor.bzl b/examples/03-complex-fixups/config/set_cfg_constructor.bzl new file mode 100644 index 000000000..7f4900d24 --- /dev/null +++ b/examples/03-complex-fixups/config/set_cfg_constructor.bzl @@ -0,0 +1,21 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +load("@prelude//cfg/modifier:cfg_constructor.bzl", "cfg_constructor_post_constraint_analysis", "cfg_constructor_pre_constraint_analysis") +load("@prelude//cfg/modifier:common.bzl", "MODIFIER_METADATA_KEY") + +def set_cfg_constructor(aliases = dict()): + project_root_cell = read_root_config("cell_aliases", "root") + current_root_cell = read_config("cell_aliases", "root") + if project_root_cell == current_root_cell: + native.set_cfg_constructor( + stage0 = cfg_constructor_pre_constraint_analysis, + stage1 = cfg_constructor_post_constraint_analysis, + key = MODIFIER_METADATA_KEY, + aliases = struct(**aliases), + extra_data = struct(), + ) diff --git a/example/project/BUCK b/examples/03-complex-fixups/project/BUCK similarity index 67% rename from example/project/BUCK rename to examples/03-complex-fixups/project/BUCK index 050f01d29..0e9f9d3af 100644 --- a/example/project/BUCK +++ b/examples/03-complex-fixups/project/BUCK @@ -4,5 +4,7 @@ rust_binary( deps = [ "//third-party:blake3", "//third-party:once_cell", + "//third-party:libc", + "//third-party:crossbeam-queue", ], ) diff --git a/example/project/test.rs b/examples/03-complex-fixups/project/test.rs similarity index 82% rename from example/project/test.rs rename to examples/03-complex-fixups/project/test.rs index 9e87acf7e..db801c066 100644 --- a/example/project/test.rs +++ b/examples/03-complex-fixups/project/test.rs @@ -7,6 +7,10 @@ use once_cell::sync::Lazy; +// Use these crates without really using them +extern crate libc; +extern crate crossbeam_queue; + const MAGIC: &str = "this is a magic string"; static SPECIAL: Lazy = Lazy::new(|| { diff --git a/example/third-party/reindeer.toml b/examples/03-complex-fixups/reindeer.toml similarity index 77% rename from example/third-party/reindeer.toml rename to examples/03-complex-fixups/reindeer.toml index 5e3a2f7cf..f12dc6d3f 100644 --- a/example/third-party/reindeer.toml +++ b/examples/03-complex-fixups/reindeer.toml @@ -8,6 +8,9 @@ ## Reindeer is under active development, and the layout and options in this file ## may change. +# Write output to third-party/BUCK +third_party_dir = "third-party" + # Parse Rust code to work out the precise set of source files for each crate. # This uses `srcfiles` which only works on Rust 2018 and without some macro # constructions. It works in almost all cases, but you may need to have a @@ -17,17 +20,12 @@ precise_srcs = true # Possible patterns for license files - lots of packages have them without # registering them in the Cargo metadata, or have more than the single file it # allows. Just look in the top-level dir for now. -license_patterns = [ - "LICENSE*", "COPYING*", "UNLICENSE*", -] +license_patterns = ["LICENSE*", "COPYING*", "UNLICENSE*"] # If a fixups.toml file is needed (eg, the package has a build.rs), then # generate a template fixups.toml to be edited. fixup_templates = true -# Emit Cargo pkg metadata into rules (experimental, not used) -#emit_metadata = false - # Include an explicit public top-level target which depends on all other library # targets, so that you can to a test/check build. include_top_level = true @@ -42,8 +40,7 @@ include_top_level = true gitignore_checksum_exclude = [".gitignore"] # Additional globs to ignore (prefer .gitignore for consistentcy with source # control) -checksum_exclude = [ -] +checksum_exclude = [] # Platforms we want to support. # @@ -64,10 +61,15 @@ checksum_exclude = [ # The platform key sets the name of the platform in `platform_deps` / # `named_platform_deps`. If it is "DEFAULT" then the platform's dependencies are # put into the normal `deps` / `named_deps`. +# +# CRUCIALLY, every [platform.xxx] key must match one in prelude/rust/cargo_package.bzl. +# If you wish to add new platforms, you will have to copy/paste the big +# DEFAULT_PLATFORM_TEMPLATES in that file and create your own wrappers (like cargo.rust_library) +# that call `apply_platform_attrs`. # x86_64-unknown-linux-gnu [platform.linux-x86_64] -x86_64-unknown-linux-gnu = [] # true for a boolean test +x86_64-unknown-linux-gnu = [] # true for a boolean test target_arch = ["x86_64"] target_endian = ["little"] target_env = ["gnu"] @@ -103,8 +105,22 @@ target_os = ["macos"] target_pointer_width = ["64"] target_vendor = ["apple"] +# macOS - aarch64-apple-darwin +# this one probably needs to be checked/expanded a bit in terms of target features +[platform.macos-arm64] +aarch64-apple-darwin = [] +target_arch = ["aarch64", "arm64"] +target_endian = ["little"] +target_env = [] +target_family = ["unix"] +target_feature = ["neon"] +target_has_atomic = ["8", "16", "32", "64", "128", "ptr"] +target_os = ["macos"] +target_pointer_width = ["64"] +target_vendor = ["apple"] + # Windows with MSVC toolchain - x86_64-pc-windows-msvc -[platform.windows] +[platform.windows-msvc] x86_64-pc-windows-msvc = [] target_arch = ["x86_64"] target_endian = ["little"] @@ -155,17 +171,17 @@ file_name = "BUCK" # default # definitions which resolve them to appropriate underlying rules # suitable for your environment. (This may also work for Buck-like # build systems such as Bazel.) -rust_library = "cargo.rust_library" # A plain Rust library -rust_binary = "cargo.rust_binary" # A Rust executable -cxx_library = "third_party_rust_cxx_library" # A C++ library (mostly for Rust -> C dependencies) -prebuilt_cxx_library = "third_party_rust_prebuilt_cxx_library" # A prebuilt library (mostly for Rust -> C dependencies) -buildscript_genrule = "buildscript_run" # Rule for running a build script to produce rustc args and generated sources +rust_library = "cargo_rust_library" # A plain Rust library +rust_binary = "cargo.rust_binary" # A Rust executable +cxx_library = "third_party_rust_cxx_library" # A C++ library (mostly for Rust -> C dependencies) +prebuilt_cxx_library = "third_party_rust_prebuilt_cxx_library" # A prebuilt library (mostly for Rust -> C dependencies) +buildscript_genrule = "buildscript_run" # Rule for running a build script to produce rustc args and generated sources # Load the macros to which the rules above will resolve. buckfile_imports = """ load("@prelude//rust:cargo_buildscript.bzl", "buildscript_run") load("@prelude//rust:cargo_package.bzl", "cargo") -load("//third-party/macros:rust_third_party.bzl", "third_party_rust_cxx_library", "third_party_rust_prebuilt_cxx_library") +load("//third-party/macros:rust_third_party.bzl", "cargo_rust_library", "third_party_rust_cxx_library", "third_party_rust_prebuilt_cxx_library") """ # Banner comment for the generated BUCK File. diff --git a/examples/03-complex-fixups/setup.sh b/examples/03-complex-fixups/setup.sh new file mode 100755 index 000000000..917803a76 --- /dev/null +++ b/examples/03-complex-fixups/setup.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +# Generate file 'third-party/BUCK'. + +set -e + +(cd ../..; cargo build) + +# Build a BUCK file to build third-party crates. +# +# This will resolve all the dependencies, and create or update +# third-party/Cargo.lock as required. +# +# It will create a template fixup.toml which you can edit as needed. You would +# typically commit these fixups and the generated third-party/BUCK in the same +# commit as above. + +../../target/debug/reindeer --third-party-dir third-party buckify diff --git a/examples/03-complex-fixups/third-party/.gitignore b/examples/03-complex-fixups/third-party/.gitignore new file mode 100644 index 000000000..cd2eec533 --- /dev/null +++ b/examples/03-complex-fixups/third-party/.gitignore @@ -0,0 +1,28 @@ +# Ignore Cargo-related stuff +.cargo +target + +# Various cruft in vendored packages +vendor/*/target +vendor/*/Cargo.lock +vendor/**/.buckconfig +vendor/**/BUCK +vendor/**/OWNERS +vendor/**/*~ +vendor/**/*.bzl +vendor/*/.github/** +vendor/*/.appveyor.yml +vendor/*/.travis.yml +/target/** + +# Bad Windows names - oh for case-insensitive regex matching! +vendor/**/[Aa][Uu][Xx] +vendor/**/[Aa][Uu][Xx].* +vendor/**/[Cc][Oo][Mm][1-9] +vendor/**/[Cc][Oo][Mm][1-9].* +vendor/**/[Cc][Oo][Nn] +vendor/**/[Cc][Oo][Nn].* +vendor/**/[Ll][Pp][Tt][1-9] +vendor/**/[Ll][Pp][Tt][1-9].* +vendor/**/[Nn][Uu][Ll] +vendor/**/[Nn][Uu][Ll].* diff --git a/examples/03-complex-fixups/third-party/Cargo.lock b/examples/03-complex-fixups/third-party/Cargo.lock new file mode 100644 index 000000000..31c8a4e3b --- /dev/null +++ b/examples/03-complex-fixups/third-party/Cargo.lock @@ -0,0 +1,99 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "blake3" +version = "1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", +] + +[[package]] +name = "cc" +version = "1.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8293772165d9345bdaaa39b45b2109591e63fe5e6fbc23c6ff930a048aa310b" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "libc" +version = "0.2.169" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "rust-third-party" +version = "0.0.0" +dependencies = [ + "blake3", + "crossbeam-queue", + "libc", + "once_cell", + "typenum", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" diff --git a/examples/03-complex-fixups/third-party/Cargo.toml b/examples/03-complex-fixups/third-party/Cargo.toml new file mode 100644 index 000000000..a6f334d2b --- /dev/null +++ b/examples/03-complex-fixups/third-party/Cargo.toml @@ -0,0 +1,25 @@ +[workspace] + +[package] +# Pseudo-package whose dependencies are imported and buckified +name = "rust-third-party" +version = "0.0.0" +edition = "2021" +publish = false + +# Dummy target to keep Cargo happy +[[bin]] +name = "top" +path = "top/main.rs" + +# List of packages to be imported, with version constraints, features +# and all options Cargo supports. +[dependencies] +blake3 = { version = "1.5.5", features = [] } +once_cell = "1.4" +libc = "0.2.169" +typenum = "1.17.0" +crossbeam-queue = "0.3.12" + +# Local patches - typically git references +[patch.crates-io] diff --git a/examples/03-complex-fixups/third-party/fixups/blake3/fixups.toml b/examples/03-complex-fixups/third-party/fixups/blake3/fixups.toml new file mode 100644 index 000000000..a00d0e0d8 --- /dev/null +++ b/examples/03-complex-fixups/third-party/fixups/blake3/fixups.toml @@ -0,0 +1,93 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under both the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree and the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. + +buildscript = [] + +## The various X86 platform fixups + +[platform_fixup.'cfg(target_arch = "x86_64")'] +cfgs = [ + "blake3_sse2_ffi", + "blake3_sse41_ffi", + "blake3_avx2_ffi", + "blake3_avx512_ffi", +] +buildscript = [] + +# , any(target_env = "fbcode", target_env = "gnu") +[[platform_fixup.'cfg(all(target_arch = "x86_64", any(target_os = "linux", target_os = "macos")))'.buildscript]] +[platform_fixup.'cfg(all(target_arch = "x86_64", any(target_os = "linux", target_os = "macos")))'.buildscript.cxx_library] +name = "simd_x86_unix" +srcs = [ + "c/blake3.c", + "c/blake3_dispatch.c", + "c/blake3_portable.c", + "c/blake3_sse2_x86-64_unix.S", + "c/blake3_sse41_x86-64_unix.S", + "c/blake3_avx2_x86-64_unix.S", + "c/blake3_avx512_x86-64_unix.S", +] +# Older versions of Clang require these flags, even for assembly. See +# https://github.com/BLAKE3-team/BLAKE3/issues/79. +compiler_flags = ["-mavx512f", "-mavx512vl"] +headers = ["c/*.h"] +compatible_with = [ + "prelude//os/constraints:linux", + "prelude//os/constraints:macos", +] + +[[platform_fixup.'cfg(all(target_arch = "x86_64", target_os = "windows", target_env = "gnu"))'.buildscript]] +[platform_fixup.'cfg(all(target_arch = "x86_64", target_os = "windows", target_env = "gnu"))'.buildscript.cxx_library] +name = "simd_x86_windows_gnu" +srcs = [ + "c/blake3.c", + "c/blake3_dispatch.c", + "c/blake3_portable.c", + "c/blake3_sse2_x86-64_windows_gnu.S", + "c/blake3_sse41_x86-64_windows_gnu.S", + "c/blake3_avx2_x86-64_windows_gnu.S", + "c/blake3_avx512_x86-64_windows_gnu.S", +] +# Older versions of Clang require these flags, even for assembly. See +# https://github.com/BLAKE3-team/BLAKE3/issues/79. +compiler_flags = ["-mavx512f", "-mavx512vl"] +headers = ["c/*.h"] +compatible_with = ["prelude//os/constraints:windows"] + +[[platform_fixup.'cfg(all(target_arch = "x86_64", target_os = "windows", target_env = "msvc"))'.buildscript]] +[platform_fixup.'cfg(all(target_arch = "x86_64", target_os = "windows", target_env = "msvc"))'.buildscript.cxx_library] +name = "simd_x86_windows_msvc" +srcs = [ + "c/blake3.c", + "c/blake3_dispatch.c", + "c/blake3_portable.c", + "c/blake3_sse2_x86-64_windows_msvc.asm", + "c/blake3_sse41_x86-64_windows_msvc.asm", + "c/blake3_avx2_x86-64_windows_msvc.asm", + "c/blake3_avx512_x86-64_windows_msvc.asm", +] +headers = ["c/*.h"] +compatible_with = ["prelude//os/constraints:windows"] + +## ARM and AArch64 fixups + +[platform_fixup.'cfg(any(target_arch = "aarch64", target_arch = "arm"))'] +cfgs = ["blake3_neon"] +buildscript = [] + +[[platform_fixup.'cfg(target_arch = "aarch64")'.buildscript]] +[platform_fixup.'cfg(target_arch = "aarch64")'.buildscript.cxx_library] +name = "simd_neon-aarch64" +srcs = ["c/blake3_neon.c"] +headers = ["c/*.h"] + +[[platform_fixup.'cfg(target_arch = "arm")'.buildscript]] +[platform_fixup.'cfg(target_arch = "arm")'.buildscript.cxx_library] +name = "simd_neon-armv7" +srcs = ["c/blake3_neon.c"] +compiler_flags = ["-mfpu=neon-vfpv4", "-mfloat-abi=hard"] +headers = ["c/*.h"] diff --git a/examples/03-complex-fixups/third-party/fixups/crossbeam-utils/fixups.toml b/examples/03-complex-fixups/third-party/fixups/crossbeam-utils/fixups.toml new file mode 100644 index 000000000..46dfe3a3a --- /dev/null +++ b/examples/03-complex-fixups/third-party/fixups/crossbeam-utils/fixups.toml @@ -0,0 +1,4 @@ +cargo_env = true + +[[buildscript]] +[buildscript.rustc_flags] diff --git a/example/third-party/fixups/libc/fixups.toml b/examples/03-complex-fixups/third-party/fixups/libc/fixups.toml similarity index 100% rename from example/third-party/fixups/libc/fixups.toml rename to examples/03-complex-fixups/third-party/fixups/libc/fixups.toml diff --git a/example/third-party/fixups/typenum/fixups.toml b/examples/03-complex-fixups/third-party/fixups/typenum/fixups.toml similarity index 100% rename from example/third-party/fixups/typenum/fixups.toml rename to examples/03-complex-fixups/third-party/fixups/typenum/fixups.toml diff --git a/examples/03-complex-fixups/third-party/macros/rust_third_party.bzl b/examples/03-complex-fixups/third-party/macros/rust_third_party.bzl new file mode 100644 index 000000000..92c0386f7 --- /dev/null +++ b/examples/03-complex-fixups/third-party/macros/rust_third_party.bzl @@ -0,0 +1,21 @@ +load("@prelude//rust:cargo_package.bzl", "cargo") + +def cargo_rust_library(name, rustc_flags = [], **kwargs): + # Always build dependencies with opt-level 1. + # This disables debug-assertions by default, so re-enable those. + # You can also do this in toolchains/BUCK by specifying global rustc_flags + # on toolchains//:rust, but this way leaves our own crates untouched. + # + rustc_flags = rustc_flags + select({ + "root//config/mode:debug": ["-Copt-level=1", "-Cdebug-assertions=true"], + "root//config/mode:release": ["-Copt-level=3"], + }) + cargo.rust_library(name = name, rustc_flags = rustc_flags, **kwargs) + +def third_party_rust_cxx_library(name, **kwargs): + # @lint-ignore BUCKLINT + native.cxx_library(name = name, **kwargs) + +def third_party_rust_prebuilt_cxx_library(name, **kwargs): + # @lint-ignore BUCKLINT + native.prebuilt_cxx_library(name = name, **kwargs) diff --git a/example/third-party/top/main.rs b/examples/03-complex-fixups/third-party/top/main.rs similarity index 100% rename from example/third-party/top/main.rs rename to examples/03-complex-fixups/third-party/top/main.rs diff --git a/examples/03-complex-fixups/toolchains/.buckconfig b/examples/03-complex-fixups/toolchains/.buckconfig new file mode 100644 index 000000000..e69de29bb diff --git a/examples/03-complex-fixups/toolchains/BUCK b/examples/03-complex-fixups/toolchains/BUCK new file mode 100644 index 000000000..c4031ed47 --- /dev/null +++ b/examples/03-complex-fixups/toolchains/BUCK @@ -0,0 +1,2 @@ +load("@prelude//toolchains:demo.bzl", "system_demo_toolchains") +system_demo_toolchains() diff --git a/src/audit_sec.rs b/src/audit_sec.rs index 13b2f022b..0d49cd4c6 100644 --- a/src/audit_sec.rs +++ b/src/audit_sec.rs @@ -8,14 +8,14 @@ use std::io::Write; use anyhow::Context; -use rustsec::advisory::Informational; -use rustsec::report::Report; -use rustsec::report::Settings; use rustsec::Database; use rustsec::Lockfile; use rustsec::Repository; use rustsec::Warning; use rustsec::WarningKind; +use rustsec::advisory::Informational; +use rustsec::report::Report; +use rustsec::report::Settings; use termcolor::Color; use termcolor::ColorChoice; use termcolor::ColorSpec; diff --git a/src/buck.rs b/src/buck.rs index f17ad9e27..e4be12c70 100644 --- a/src/buck.rs +++ b/src/buck.rs @@ -18,10 +18,10 @@ use std::io::Write; use std::path::PathBuf; use semver::Version; +use serde::Serialize; use serde::ser::SerializeMap; use serde::ser::SerializeSeq; use serde::ser::Serializer; -use serde::Serialize; use serde_starlark::FunctionCall; use crate::collection::SelectSet; @@ -922,12 +922,9 @@ impl Serialize for CxxLibrary { map.serialize_entry("compiler_flags", compiler_flags)?; } if include_directories.iter().any(SubtargetOrPath::is_path) { - map.serialize_entry( - "include_directories", - &IncludeDirectories { - include_directories, - }, - )?; + map.serialize_entry("include_directories", &IncludeDirectories { + include_directories, + })?; } if !licenses.is_empty() { map.serialize_entry("licenses", licenses)?; @@ -940,13 +937,10 @@ impl Serialize for CxxLibrary { .iter() .any(SubtargetOrPath::is_subtarget) { - map.serialize_entry( - "preprocessor_flags", - &PreprocessorFlags { - include_directories, - preprocessor_flags, - }, - )?; + map.serialize_entry("preprocessor_flags", &PreprocessorFlags { + include_directories, + preprocessor_flags, + })?; } if *undefined_symbols { map.serialize_entry("undefined_symbols", undefined_symbols)?; diff --git a/src/buckify.rs b/src/buckify.rs index ff4bf3137..a73243bce 100644 --- a/src/buckify.rs +++ b/src/buckify.rs @@ -19,16 +19,18 @@ use std::io::Write; use std::path::Component; use std::path::Path; use std::path::PathBuf; -use std::sync::mpsc; use std::sync::Mutex; +use std::sync::mpsc; -use anyhow::bail; use anyhow::Context; +use anyhow::bail; use cached::proc_macro::cached; use fnv::FnvHasher; use itertools::Itertools; use url::Url; +use crate::Args; +use crate::Paths; use crate::buck; use crate::buck::Alias; use crate::buck::BuckPath; @@ -47,7 +49,6 @@ use crate::buck::RustLibrary; use crate::buck::StringOrPath; use crate::buck::SubtargetOrPath; use crate::buck::Visibility; -use crate::cargo::cargo_get_lockfile_and_metadata; use crate::cargo::ArtifactKind; use crate::cargo::Edition; use crate::cargo::Manifest; @@ -55,6 +56,7 @@ use crate::cargo::ManifestTarget; use crate::cargo::PkgId; use crate::cargo::Source; use crate::cargo::TargetReq; +use crate::cargo::cargo_get_lockfile_and_metadata; use crate::collection::SetOrMap; use crate::config::Config; use crate::config::VendorConfig; @@ -65,13 +67,11 @@ use crate::glob::NO_EXCLUDE; use crate::index; use crate::lockfile::Lockfile; use crate::lockfile::LockfilePackage; -use crate::platform::platform_names_for_expr; use crate::platform::PlatformExpr; use crate::platform::PlatformName; +use crate::platform::platform_names_for_expr; use crate::srcfiles::crate_srcfiles; use crate::universe::UniverseName; -use crate::Args; -use crate::Paths; // normalize a/b/../c => a/c and a/./b => a/b pub fn normalize_path(path: &Path) -> PathBuf { @@ -209,15 +209,22 @@ fn generate_rules<'scope>( if !matching_kind { continue; } - match generate_target_rules(context, pkg, tgt) { + + let will_use_rules = { + let is_private_root_pkg = + context.index.is_root_package(pkg) && !context.index.is_public_package(pkg); + let is_ignored_workspace_package = !context.config.include_workspace_members + && context.index.is_workspace_package(&pkg); + !is_private_root_pkg && !is_ignored_workspace_package + }; + + match generate_target_rules(context, pkg, tgt, will_use_rules) { Ok((rules, _)) if rules.is_empty() => { // Don't generate rules for dependencies if we're not emitting // any rules for this target. } Ok((rules, mut deps)) => { - let is_private_root_pkg = - context.index.is_root_package(pkg) && !context.index.is_public_package(pkg); - if !is_private_root_pkg { + if will_use_rules { for rule in rules { let _ = rule_tx.send(Ok(rule)); } @@ -418,6 +425,7 @@ fn generate_target_rules<'scope>( context: &'scope RuleContext<'scope>, pkg: &'scope Manifest, tgt: &'scope ManifestTarget, + will_use_rules: bool, ) -> anyhow::Result<(Vec, Vec<(&'scope Manifest, TargetReq<'scope>)>)> { let RuleContext { config, @@ -428,7 +436,7 @@ fn generate_target_rules<'scope>( log::debug!("Generating rules for package {} target {}", pkg, tgt.name); - let fixups = Fixups::new(config, paths, index, pkg, tgt)?; + let fixups = Fixups::new(config, paths, index, pkg, tgt, will_use_rules)?; if fixups.omit_target() { return Ok((vec![], vec![])); @@ -440,7 +448,19 @@ fn generate_target_rules<'scope>( let mapped_manifest_dir = if matches!(config.vendor, VendorConfig::Source(_)) || matches!(pkg.source, Source::Local) { - relative_path(&paths.third_party_dir, manifest_dir) + match manifest_dir + .strip_prefix(&paths.third_party_dir) + .with_context(|| format!( + "crate sources would be inaccessible from the generated BUCK file, cannot refer to {} from {}.", + relative_path(&paths.third_party_dir, manifest_dir).display(), + paths.third_party_dir.join(&config.buck.file_name).display(), + )) + .map(ToOwned::to_owned){ + Err(_) if !will_use_rules => { + PathBuf::from("__unused__") + } + res => res? + } } else if let VendorConfig::LocalRegistry = config.vendor { PathBuf::from(format!("{}-{}.crate", pkg.name, pkg.version)) } else if let Source::Git { repo, .. } = &pkg.source { @@ -1015,15 +1035,10 @@ fn buckify_for_universe( measure_time::info_time!("Generating buck rules"); rayon::scope(move |scope| { for &workspace_member in &context.index.workspace_members { - generate_dep_rules( - context, - scope, - tx.clone(), - [ - (workspace_member, TargetReq::Lib), - (workspace_member, TargetReq::EveryBin), - ], - ); + generate_dep_rules(context, scope, tx.clone(), [ + (workspace_member, TargetReq::Lib), + (workspace_member, TargetReq::EveryBin), + ]); } }); } diff --git a/src/cargo.rs b/src/cargo.rs index 13b8a02eb..158a4b55e 100644 --- a/src/cargo.rs +++ b/src/cargo.rs @@ -24,17 +24,17 @@ use std::process::Stdio; use std::thread; use anyhow::Context; -use serde::de::DeserializeOwned; use serde::Deserialize; use serde::Deserializer; use serde::Serialize; +use serde::de::DeserializeOwned; +use crate::Args; +use crate::Paths; use crate::config::Config; use crate::config::VendorConfig; use crate::lockfile::Lockfile; use crate::platform::PlatformExpr; -use crate::Args; -use crate::Paths; pub fn cargo_get_lockfile_and_metadata( config: &Config, @@ -80,14 +80,8 @@ pub fn cargo_get_lockfile_and_metadata( lockfile = Some(Lockfile::load(paths)?); }; - let metadata: Metadata = run_cargo_json( - config, - cargo_home, - &paths.third_party_dir, - args, - &cargo_flags, - ) - .context("parsing metadata")?; + let metadata: Metadata = + run_cargo_json(config, cargo_home, None, args, &cargo_flags).context("parsing metadata")?; let lockfile = match lockfile { Some(existing_lockfile) => existing_lockfile, @@ -101,7 +95,7 @@ pub fn cargo_get_lockfile_and_metadata( pub(crate) fn run_cargo( config: &Config, cargo_home: Option<&Path>, - current_dir: &Path, + current_dir: Option<&Path>, args: &Args, opts: &[&str], ) -> anyhow::Result> { @@ -118,15 +112,15 @@ pub(crate) fn run_cargo( } log::debug!( - "Running Cargo command {:?} in {}", + "Running Cargo command {:?} in dir {:?}", cmdline, - current_dir.display() + current_dir ); let mut cargo_command = if let Some(cargo_path) = args.cargo_path.as_ref() { Command::new(cargo_path) } else if let Some(bin) = config.cargo.cargo.as_ref() { - Command::new(config.config_path.join(bin)) + Command::new(config.config_dir.join(bin)) } else { Command::new("cargo") }; @@ -149,15 +143,18 @@ pub(crate) fn run_cargo( } } } else if let Some(bin) = config.cargo.rustc.as_ref() { - cargo_command.env("RUSTC", config.config_path.join(bin)); + cargo_command.env("RUSTC", config.config_dir.join(bin)); } if let Some(cargo_home) = cargo_home { cargo_command.env("CARGO_HOME", cargo_home); } + if let Some(current_dir) = current_dir { + cargo_command.current_dir(current_dir); + } + cargo_command - .current_dir(current_dir) .args(&cmdline) .envs(envs) .stdout(Stdio::piped()) @@ -208,7 +205,7 @@ pub(crate) fn run_cargo( pub(crate) fn run_cargo_json( config: &Config, cargo_home: Option<&Path>, - current_dir: &Path, + current_dir: Option<&Path>, args: &Args, opts: &[&str], ) -> anyhow::Result { @@ -713,8 +710,8 @@ fn parse_source(source: &str) -> Option { #[cfg(test)] mod test { - use super::parse_source; use super::Source; + use super::parse_source; #[test] fn test_parses_source_git() { diff --git a/src/cfg.rs b/src/cfg.rs index 05f3c2223..d54221b45 100644 --- a/src/cfg.rs +++ b/src/cfg.rs @@ -5,6 +5,7 @@ * LICENSE file in the root directory of this source tree. */ +use nom::IResult; use nom::branch::alt; use nom::bytes::complete::escaped; use nom::bytes::complete::tag; @@ -18,9 +19,9 @@ use nom::combinator::map; use nom::combinator::opt; use nom::combinator::recognize; use nom::combinator::verify; -use nom::error::context; use nom::error::ContextError; use nom::error::ParseError; +use nom::error::context; use nom::multi::many0_count; use nom::multi::separated_list0; use nom::multi::separated_list1; @@ -29,7 +30,6 @@ use nom::sequence::pair; use nom::sequence::preceded; use nom::sequence::separated_pair; use nom::sequence::terminated; -use nom::IResult; use unicode_ident::is_xid_continue; use unicode_ident::is_xid_start; @@ -209,13 +209,10 @@ mod test { println!("res = {:?}", res); assert_eq!( res, - Ok(( - "", - Value { - key: "feature", - value: "bloop" - } - )) + Ok(("", Value { + key: "feature", + value: "bloop" + })) ) } @@ -225,13 +222,10 @@ mod test { println!("res = {:?}", res); assert_eq!( res, - Ok(( - "", - Value { - key: "target_env", - value: "" - } - )) + Ok(("", Value { + key: "target_env", + value: "" + })) ) } diff --git a/src/collection.rs b/src/collection.rs index 24a5b6bf7..f597faf38 100644 --- a/src/collection.rs +++ b/src/collection.rs @@ -10,13 +10,13 @@ use std::collections::BTreeSet; use std::fmt; use std::marker::PhantomData; -use serde::de::value::MapAccessDeserializer; -use serde::de::value::SeqAccessDeserializer; use serde::de::Deserialize; use serde::de::Deserializer; use serde::de::MapAccess; use serde::de::SeqAccess; use serde::de::Visitor; +use serde::de::value::MapAccessDeserializer; +use serde::de::value::SeqAccessDeserializer; use serde::ser::Serialize; use serde::ser::SerializeSeq; use serde::ser::SerializeTupleStruct; diff --git a/src/config.rs b/src/config.rs index 91658527d..5a1ce81f2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -24,12 +24,12 @@ use std::path::PathBuf; use anyhow::Context; use monostate::MustBe; -use serde::de::value::MapAccessDeserializer; +use serde::Deserialize; +use serde::Serialize; use serde::de::Deserializer; use serde::de::MapAccess; use serde::de::Visitor; -use serde::Deserialize; -use serde::Serialize; +use serde::de::value::MapAccessDeserializer; use crate::platform::PlatformConfig; use crate::platform::PlatformName; @@ -41,7 +41,17 @@ use crate::universe::UniverseName; pub struct Config { /// Path the config was read from #[serde(skip)] - pub config_path: PathBuf, + pub config_dir: PathBuf, + + /// Path to the Cargo.toml we are buckifying + #[serde(default)] + pub manifest_path: Option, + + /// Where to write the output. + /// + /// (Default = "", i.e. current directory) + #[serde(default)] + pub third_party_dir: Option, /// Try to compute a precise list of sources rather than using globbing #[serde(default)] @@ -67,6 +77,14 @@ pub struct Config { #[serde(default)] pub include_top_level: bool, + /// Include workspace members in the generated BUCK file. + /// + /// Generlly workspace members are located outside the third party directory. + /// So this is probably only relevant if you are keeping a workspace + /// Cargo.toml and its generated BUCK file in the same directory. + #[serde(default)] + pub include_workspace_members: bool, + /// Use strict glob matching #[serde(default)] pub strict_globs: bool, @@ -317,11 +335,13 @@ where deserializer.deserialize_any(VendorConfigVisitor) } -pub fn read_config(dir: &Path) -> anyhow::Result { - let reindeer_toml = dir.join("reindeer.toml"); +pub fn read_config(reindeer_toml: &Path) -> anyhow::Result { + let dir = reindeer_toml + .parent() + .context("Invalid path to reindeer.toml")?; let mut config = try_read_config(&reindeer_toml)?; - config.config_path = dir.to_path_buf(); + config.config_dir = dir.to_path_buf(); if config.buck.buckfile_imports.is_default { // Fill in some prelude imports so Reindeer generates working targets diff --git a/src/fixups.rs b/src/fixups.rs index 66420762f..f2b333971 100644 --- a/src/fixups.rs +++ b/src/fixups.rs @@ -16,10 +16,11 @@ use std::fs; use std::path::Path; use std::path::PathBuf; +use anyhow::Context; use anyhow::anyhow; use anyhow::bail; -use anyhow::Context; +use crate::Paths; use crate::buck; use crate::buck::Alias; use crate::buck::BuckPath; @@ -44,14 +45,13 @@ use crate::collection::SetOrMap; use crate::config::Config; use crate::config::VendorConfig; use crate::glob::Globs; -use crate::glob::SerializableGlobSet as GlobSet; use crate::glob::NO_EXCLUDE; +use crate::glob::SerializableGlobSet as GlobSet; use crate::index::Index; use crate::index::ResolvedDep; -use crate::platform::platform_names_for_expr; use crate::platform::PlatformExpr; use crate::platform::PlatformPredicate; -use crate::Paths; +use crate::platform::platform_names_for_expr; mod buildscript; mod config; @@ -98,6 +98,7 @@ impl<'meta> Fixups<'meta> { index: &'meta Index, package: &'meta Manifest, target: &'meta ManifestTarget, + will_use_rules: bool, ) -> anyhow::Result { let fixup_dir = paths.third_party_dir.join("fixups").join(&package.name); let fixup_path = fixup_dir.join("fixups.toml"); @@ -108,7 +109,8 @@ impl<'meta> Fixups<'meta> { } else { log::debug!("no fixups at {}", fixup_path.display()); let fixup = FixupConfigFile::template(&paths.third_party_dir, target); - if config.fixup_templates && target.kind_custom_build() { + // will_use_rules: avoid writing for e.g. `include_workspace_members = false` + if config.fixup_templates && target.kind_custom_build() && will_use_rules { log::debug!( "Writing template for {} to {}", package, diff --git a/src/fixups/buildscript.rs b/src/fixups/buildscript.rs index 1bb88e369..de87438e2 100644 --- a/src/fixups/buildscript.rs +++ b/src/fixups/buildscript.rs @@ -11,14 +11,14 @@ use std::marker::PhantomData; use std::ops::Deref; use std::path::PathBuf; -use serde::de::Error as DeError; -use serde::de::MapAccess; -use serde::de::Visitor; -use serde::ser::SerializeMap; use serde::Deserialize; use serde::Deserializer; use serde::Serialize; use serde::Serializer; +use serde::de::Error as DeError; +use serde::de::MapAccess; +use serde::de::Visitor; +use serde::ser::SerializeMap; use crate::cargo::TargetKind; use crate::collection::SetOrMap; diff --git a/src/fixups/config.rs b/src/fixups/config.rs index c291c57e8..1ed0f5c01 100644 --- a/src/fixups/config.rs +++ b/src/fixups/config.rs @@ -12,13 +12,13 @@ use std::fmt; use std::path::Path; use std::path::PathBuf; -use serde::de::value::SeqAccessDeserializer; -use serde::de::SeqAccess; -use serde::de::Visitor; use serde::Deserialize; use serde::Deserializer; use serde::Serialize; use serde::Serializer; +use serde::de::SeqAccess; +use serde::de::Visitor; +use serde::de::value::SeqAccessDeserializer; use strum::IntoEnumIterator as _; use walkdir::WalkDir; diff --git a/src/index.rs b/src/index.rs index a3ddfd044..3b3b99e57 100644 --- a/src/index.rs +++ b/src/index.rs @@ -10,6 +10,7 @@ use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::HashMap; +use std::collections::HashSet; use anyhow::Context as _; @@ -35,6 +36,8 @@ pub struct Index<'meta> { pub root_pkg: Option<&'meta Manifest>, /// All packages considered part of the workspace. pub workspace_members: Vec<&'meta Manifest>, + /// Faster lookup for workspace members + workspace_packages: HashSet<&'meta PkgId>, /// Set of package IDs from which at least one target is public mapped to an optional rename public_packages: BTreeMap<&'meta PkgId, Option<&'meta str>>, /// The (possibly renamed) names of all packages which have at least one @@ -77,7 +80,7 @@ impl<'meta> Index<'meta> { None }; - let workspace_members = metadata + let workspace_members: Vec<_> = metadata .workspace_default_members .iter() .filter_map(|pkgid| pkgid_to_pkg.get(pkgid).copied()) @@ -87,6 +90,7 @@ impl<'meta> Index<'meta> { pkgid_to_pkg, pkgid_to_node: metadata.resolve.nodes.iter().map(|n| (&n.id, n)).collect(), root_pkg, + workspace_packages: workspace_members.iter().map(|x| &x.id).collect(), workspace_members, public_packages: BTreeMap::new(), public_package_names: BTreeSet::new(), @@ -153,6 +157,11 @@ impl<'meta> Index<'meta> { self.public_packages.contains_key(&pkg.id) } + /// Test if this is a workspace member + pub fn is_workspace_package(&self, pkg: &Manifest) -> bool { + self.workspace_packages.contains(&pkg.id) + } + /// Test if there is any target from any package with the given (possibly /// renamed) crate name which is public. pub fn is_public_package_name(&self, name: &str) -> bool { diff --git a/src/lockfile.rs b/src/lockfile.rs index 8d4885b46..93d340a86 100644 --- a/src/lockfile.rs +++ b/src/lockfile.rs @@ -11,9 +11,9 @@ use anyhow::Context; use serde::Deserialize; use serde::Deserializer; +use crate::Paths; use crate::cargo::Manifest; use crate::cargo::Source; -use crate::Paths; #[derive(Deserialize, Debug)] pub struct Lockfile { diff --git a/src/main.rs b/src/main.rs index 4c08179ba..e95e23f33 100644 --- a/src/main.rs +++ b/src/main.rs @@ -20,8 +20,10 @@ //! //! (TBD - rest of it) +use std::path::Path; use std::path::PathBuf; +use anyhow::Context; use clap::Parser; use clap::Subcommand; @@ -56,11 +58,23 @@ pub struct Args { /// Extra cargo options #[arg(long, value_name = "ARGUMENT")] cargo_options: Vec, - /// Path to third-party dir - #[arg(long, default_value = ".", value_name = "PATH")] - third_party_dir: PathBuf, + /// Path to third-party dir. Overrides configuration from `reindeer.toml`. + /// + /// Path is relative to current dir, whereas the one in `reindeer.toml` is relative to the + /// parent directory of the `reindeer.toml` file. + #[arg(long, value_name = "PATH")] + third_party_dir: Option, #[command(subcommand)] subcommand: SubCommand, + /// Path to the `Cargo.toml to generate from. Overrides configuration from `reindeer.toml`. + /// + /// Path is relative to current dir, whereas the one in `reindeer.toml` is relative to the + /// parent directory of the `reindeer.toml` file. + #[arg(long, value_name = "PATH")] + manifest_path: Option, + /// Path to a `reindeer.toml` file to read configuration from. + #[arg(short = 'c', long, value_name = "PATH")] + config: Option, } #[derive(Debug, Subcommand)] @@ -107,15 +121,74 @@ pub struct Paths { fn try_main() -> anyhow::Result<()> { let args = Args::parse(); - let third_party_dir = dunce::canonicalize(&args.third_party_dir)?; - let mut config = config::read_config(&third_party_dir)?; + let paths; + let mut config; + + if let Some(from_args) = args.third_party_dir.as_deref() { + let third_party_dir = dunce::canonicalize(&from_args)?; + config = config::read_config(&third_party_dir.join("reindeer.toml"))?; + + let manifest_path = args + .manifest_path + .clone() + .unwrap_or_else(|| third_party_dir.join("Cargo.toml")); + + paths = Paths { + lockfile_path: manifest_path.with_file_name("Cargo.lock"), + manifest_path, + cargo_home: third_party_dir.join(".cargo"), + third_party_dir, + }; + } else { + let config_path = args.config.as_deref().unwrap_or(Path::new("reindeer.toml")); + config = config::read_config(config_path)?; + + // These unwrap sequences look gnarly, but essentially + // - `reindeer` (no flags) == `reindeer -c reindeer.toml` + // - config.third_party_dir and config.manifest_path are new, and many projects + // don't have them + // - third_party_dir defaulted to . before, and remains so + // - manifest_path was not configurable before, so absent configuration remains + // {third_party_dir}/Cargo.toml + let third_party_dir = args + .third_party_dir + .clone() + .or_else(|| Some(config.config_dir.join(config.third_party_dir.as_deref()?))) + .unwrap_or_else(|| ".".into()); + let third_party_dir = dunce::canonicalize(&third_party_dir)?; + + let manifest_path = args + .manifest_path + .clone() + .or_else(|| Some(config.config_dir.join(config.manifest_path.as_deref()?))) + .unwrap_or_else(|| third_party_dir.join("Cargo.toml")); + + let lockfile_path = manifest_path.with_file_name("Cargo.lock"); + + paths = Paths { + lockfile_path, + manifest_path, + cargo_home: third_party_dir.join(".cargo"), + third_party_dir, + }; + } - let paths = Paths { - manifest_path: third_party_dir.join("Cargo.toml"), - lockfile_path: third_party_dir.join("Cargo.lock"), - cargo_home: third_party_dir.join(".cargo"), - third_party_dir, - }; + if !paths.manifest_path.is_file() { + return Err(anyhow::anyhow!( + "Path {} is not a file", + paths.manifest_path.display() + )); + } + + if !paths.third_party_dir.exists() { + std::fs::create_dir_all(&paths.third_party_dir) + .with_context(|| format!("Creating directory {}", paths.third_party_dir.display()))?; + } else if !paths.third_party_dir.is_dir() { + return Err(anyhow::anyhow!( + "Path {} must be a directory", + paths.manifest_path.display() + )); + } log::debug!("Args = {:#?}, paths {:#?}", args, paths); @@ -133,17 +206,11 @@ fn try_main() -> anyhow::Result<()> { } SubCommand::Update { .. } => { - let _ = cargo::run_cargo( - &config, - Some(&paths.cargo_home), - &paths.third_party_dir, - &args, - &[ - "generate-lockfile", - "--manifest-path", - paths.manifest_path.to_str().unwrap(), - ], - )?; + let _ = cargo::run_cargo(&config, Some(&paths.cargo_home), None, &args, &[ + "generate-lockfile", + "--manifest-path", + paths.manifest_path.to_str().unwrap(), + ])?; } SubCommand::Buckify { stdout } => { diff --git a/src/platform.rs b/src/platform.rs index e4a0774d0..bf6a6d334 100644 --- a/src/platform.rs +++ b/src/platform.rs @@ -11,8 +11,8 @@ use std::error; use std::fmt; use std::fmt::Display; -use nom::error::convert_error; use nom::error::VerboseError; +use nom::error::convert_error; use serde::Deserialize; use serde::Serialize; diff --git a/src/srcfiles.rs b/src/srcfiles.rs index cf4e1d21c..25d0c04ed 100644 --- a/src/srcfiles.rs +++ b/src/srcfiles.rs @@ -191,13 +191,12 @@ impl SourceFinder<'_> { } Err(err) if err.kind() == io::ErrorKind::NotFound => false, Err(err) => { - self.sources.errors.push(Error::new( - self.current, - ErrorKind::FileError { + self.sources + .errors + .push(Error::new(self.current, ErrorKind::FileError { source_path: source_path.to_owned(), source: err, - }, - )); + })); true } } @@ -406,8 +405,8 @@ impl syn::parse::Parse for CfgIf { mod tests { use std::io; - use tempfile::tempdir; use tempfile::TempDir; + use tempfile::tempdir; use super::*; diff --git a/src/universe.rs b/src/universe.rs index da3bc3d45..ac9a7c810 100644 --- a/src/universe.rs +++ b/src/universe.rs @@ -252,10 +252,10 @@ pub fn mutate_manifest( config: &UniverseConfig, path: &Path, ) -> anyhow::Result { - use toml_edit::visit_mut::visit_table_like_kv_mut; - use toml_edit::visit_mut::VisitMut; use toml_edit::Item; use toml_edit::Value; + use toml_edit::visit_mut::VisitMut; + use toml_edit::visit_mut::visit_table_like_kv_mut; let original_contents = std::fs::read_to_string(path) .with_context(|| format!("reading manifest {}", path.display()))?; diff --git a/src/vendor.rs b/src/vendor.rs index 693e00123..e23576ab8 100644 --- a/src/vendor.rs +++ b/src/vendor.rs @@ -18,6 +18,8 @@ use indexmap::IndexMap; use serde::Deserialize; use serde::Serialize; +use crate::Args; +use crate::Paths; use crate::buckify::relative_path; use crate::cargo; use crate::config::Config; @@ -25,8 +27,6 @@ use crate::config::VendorConfig; use crate::config::VendorSourceConfig; use crate::remap::RemapConfig; use crate::remap::RemapSource; -use crate::Args; -use crate::Paths; #[derive(Debug, Deserialize, Serialize)] struct CargoChecksums { @@ -43,42 +43,33 @@ pub(crate) fn cargo_vendor( paths: &Paths, ) -> anyhow::Result<()> { let vendordir = Path::new("vendor"); // relative to third_party_dir + let full_vendor_dir = paths.third_party_dir.join("vendor"); if let VendorConfig::LocalRegistry = config.vendor { let mut cmdline = vec![ "local-registry", "-s", paths.lockfile_path.to_str().unwrap(), - vendordir.to_str().unwrap(), + full_vendor_dir.to_str().unwrap(), "--git", ]; if no_delete { cmdline.push("--no-delete"); } log::info!("Running cargo {:?}", cmdline); - let _ = cargo::run_cargo( - config, - Some(&paths.cargo_home), - &paths.third_party_dir, - args, - &cmdline, - )?; + let _ = cargo::run_cargo(config, Some(&paths.cargo_home), None, args, &cmdline)?; let mut remap = RemapConfig::default(); - remap.sources.insert( - "crates-io".to_owned(), - RemapSource { - registry: Some("sparse+https://index.crates.io/".to_owned()), - replace_with: Some("local-registry".to_owned()), - ..RemapSource::default() - }, - ); - remap.sources.insert( - "local-registry".to_owned(), - RemapSource { + remap.sources.insert("crates-io".to_owned(), RemapSource { + registry: Some("sparse+https://index.crates.io/".to_owned()), + replace_with: Some("local-registry".to_owned()), + ..RemapSource::default() + }); + remap + .sources + .insert("local-registry".to_owned(), RemapSource { local_registry: Some(vendordir.to_owned()), ..RemapSource::default() - }, - ); + }); let config_toml = toml::to_string(&remap).context("failed to serialize config.toml")?; fs::write(paths.cargo_home.join("config.toml"), config_toml)?; assert!(is_vendored(config, paths)?); @@ -87,7 +78,7 @@ pub(crate) fn cargo_vendor( "vendor", "--manifest-path", paths.manifest_path.to_str().unwrap(), - vendordir.to_str().unwrap(), + full_vendor_dir.to_str().unwrap(), "--versioned-dirs", ]; if no_delete { @@ -97,13 +88,7 @@ pub(crate) fn cargo_vendor( fs::create_dir_all(&paths.cargo_home)?; log::info!("Running cargo {:?}", cmdline); - let cargoconfig = cargo::run_cargo( - config, - Some(&paths.cargo_home), - &paths.third_party_dir, - args, - &cmdline, - )?; + let cargoconfig = cargo::run_cargo(config, Some(&paths.cargo_home), None, args, &cmdline)?; fs::write(paths.cargo_home.join("config.toml"), &cargoconfig)?; if !cargoconfig.is_empty() {