Add differential testing harness

.cargo/config.toml

@@ -1,5 +1,6 @@
 [alias]
 compiletest = "run --release -p compiletests --"
+difftest = "run --release -p difftests --"
 
 
 [target.x86_64-pc-windows-msvc]

.github/workflows/ci.yaml

@@ -42,6 +42,13 @@ jobs:
       - if: ${{ runner.os == 'Linux' }}
         name: Linux - Install native dependencies
         run: sudo apt install libwayland-cursor0 libxkbcommon-dev libwayland-dev
+      - if: ${{ runner.os == 'Linux' }}
+        name: Install xvfb, llvmpipe and lavapipe
+        run: |
+          sudo apt-get update -y -qq
+          sudo add-apt-repository ppa:kisak/turtle -y
+          sudo apt-get update
+          sudo apt install -y xvfb libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers
       # cargo version is a random command that forces the installation of rust-toolchain
       - name: install rust-toolchain
         run: cargo version
@@ -64,6 +71,10 @@ jobs:
         if: ${{ matrix.target != 'aarch64-linux-android' }}
         run: cargo run -p compiletests --release --no-default-features --features "use-installed-tools" -- --target-env vulkan1.1,vulkan1.2,spv1.3
 
+      - name: difftest
+        if: ${{ matrix.target != 'aarch64-linux-android' }}
+        run: cargo run -p difftests --release --no-default-features --features "use-installed-tools"
+
       - name: workspace test
         if: ${{ matrix.target != 'aarch64-linux-android' }}
         run: cargo test --release
@@ -147,6 +158,8 @@ jobs:
         run: cargo fmt --all -- --check
       - name: Rustfmt compiletests
         run: shopt -s globstar && rustfmt --check tests/compiletests/ui/**/*.rs
+      - name: Rustfmt difftests
+        run: cargo fmt --check --all --manifest-path tests/difftests/tests/Cargo.toml
       - name: Check docs are valid
         run: RUSTDOCFLAGS=-Dwarnings cargo doc --no-deps
       - name: Check docs for `spirv-std` and `spirv-builder` on stable (for docs.rs)

Cargo.toml

@@ -27,6 +27,8 @@ members = [
 
     "tests/compiletests",
     "tests/compiletests/deps-helper",
+    "tests/difftests/bin",
+    "tests/difftests/lib",
 ]
 
 [workspace.package]

crates/spirv-builder/Cargo.toml

@@ -46,6 +46,7 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 thiserror = "2.0.12"
 semver = { version = "1.0.24", features = ["serde"] }
+cargo_metadata = "0.19.2"
 
 notify = { version = "7.0", optional = true }
 # Pinning clap, as newer versions have raised min rustc version without being marked a breaking change

crates/spirv-builder/src/lib.rs

@@ -128,6 +128,8 @@ pub enum SpirvBuilderError {
     MetadataFileMissing(#[from] std::io::Error),
     #[error("unable to parse multi-module metadata file")]
     MetadataFileMalformed(#[from] serde_json::Error),
+    #[error("cargo metadata error")]
+    CargoMetadata(#[from] cargo_metadata::Error),
 }
 
 const SPIRV_TARGET_PREFIX: &str = "spirv-unknown-";
@@ -427,10 +429,11 @@ pub struct SpirvBuilder {
     /// [this RFC](https://rust-lang.github.io/rfcs/0131-target-specification.html).
     #[cfg_attr(feature = "clap", clap(skip))]
     pub path_to_target_spec: Option<PathBuf>,
-    /// Set the target dir path within `./target` to use for building shaders. Defaults to `spirv-builder`, resulting
-    /// in the path `./target/spirv-builder`.
+    /// Set the target dir path to use for building shaders. Relative paths will be resolved
+    /// relative to the `target` dir of the shader crate, absolute paths are used as is.
+    /// Defaults to `spirv-builder`, resulting in the path `./target/spirv-builder`.
     #[cfg_attr(feature = "clap", clap(skip))]
-    pub target_dir_path: Option<String>,
+    pub target_dir_path: Option<PathBuf>,
 
     // `rustc_codegen_spirv::linker` codegen args
     /// Change the shader `panic!` handling strategy (see [`ShaderPanicStrategy`]).
@@ -648,10 +651,11 @@ impl SpirvBuilder {
         self
     }
 
-    /// Set the target dir path within `./target` to use for building shaders. Defaults to `spirv-builder`, resulting
-    /// in the path `./target/spirv-builder`.
+    /// Set the target dir path to use for building shaders. Relative paths will be resolved
+    /// relative to the `target` dir of the shader crate, absolute paths are used as is.
+    /// Defaults to `spirv-builder`, resulting in the path `./target/spirv-builder`.
     #[must_use]
-    pub fn target_dir_path(mut self, name: impl Into<String>) -> Self {
+    pub fn target_dir_path(mut self, name: impl Into<PathBuf>) -> Self {
         self.target_dir_path = Some(name.into());
         self
     }
@@ -932,34 +936,21 @@ fn invoke_rustc(builder: &SpirvBuilder) -> Result<PathBuf, SpirvBuilderError> {
         rustflags.extend(extra_rustflags.split_whitespace().map(|s| s.to_string()));
     }
 
-    // If we're nested in `cargo` invocation, use a different `--target-dir`,
-    // to avoid waiting on the same lock (which effectively dead-locks us).
-    let outer_target_dir = match (env::var("PROFILE"), env::var_os("OUT_DIR")) {
-        (Ok(outer_profile), Some(dir)) => {
-            // Strip `$outer_profile/build/*/out`.
-            [&outer_profile, "build", "*", "out"].iter().rev().try_fold(
-                PathBuf::from(dir),
-                |mut dir, &filter| {
-                    if (filter == "*" || dir.ends_with(filter)) && dir.pop() {
-                        Some(dir)
-                    } else {
-                        None
-                    }
-                },
-            )
-        }
-        _ => None,
+    let target_dir_path = builder
+        .target_dir_path
+        .clone()
+        .unwrap_or_else(|| PathBuf::from("spirv-builder"));
+    let target_dir = if target_dir_path.is_absolute() {
+        target_dir_path
+    } else {
+        let metadata = cargo_metadata::MetadataCommand::new()
+            .current_dir(path_to_crate)
+            .exec()?;
+        metadata
+            .target_directory
+            .into_std_path_buf()
+            .join(target_dir_path)
     };
-    // FIXME(eddyb) use `crate metadata` to always be able to get the "outer"
-    // (or "default") `--target-dir`, to append `/spirv-builder` to it.
-    let target_dir = outer_target_dir.map(|outer| {
-        outer.join(
-            builder
-                .target_dir_path
-                .as_deref()
-                .unwrap_or("spirv-builder"),
-        )
-    });
 
     let profile = if builder.release { "release" } else { "dev" };
 
@@ -1014,10 +1005,7 @@ fn invoke_rustc(builder: &SpirvBuilder) -> Result<PathBuf, SpirvBuilderError> {
             .arg(builder.shader_crate_features.features.join(","));
     }
 
-    // NOTE(eddyb) see above how this is computed and why it might be missing.
-    if let Some(target_dir) = target_dir {
-        cargo.arg("--target-dir").arg(target_dir);
-    }
+    cargo.arg("--target-dir").arg(target_dir);
 
     // Clear Cargo environment variables that we don't want to leak into the
     // inner invocation of Cargo (because e.g. build scripts might read them),

deny.toml

@@ -60,6 +60,8 @@ skip-tree = [
     { name = "num_cpus", version = "=1.16.0", depth = 2 },
     # HACK(LegNeato) `tracing-tree` uses newer dependencies of `tracing`.
     { name = "tracing-tree", version = "=0.3.1" },
+    # HACK(LegNeato) `thorin` has not yet released the version that bumps this.
+    { name = "gimli", version = "=0.30.0" },
 ]
 
 

docs/src/testing.md

@@ -1,14 +1,18 @@
 # Testing Rust-GPU
 
 Rust-GPU has a couple of different kinds of tests, most can be ran through
-`cargo test`, however Rust-GPU also has end-to-end tests for compiling Rust and
-validating its SPIR-V output, which can ran by running `cargo compiletest`.
+`cargo test`. Rust-GPU also has end-to-end tests for compiling Rust and
+validating its SPIR-V output, which can ran by running `cargo compiletest`. Finally,
+Rust-GPU has differential tests, which runs Rust and WGSL shaders and
+makes sure they have the same output. These can be run with `cargo difftest`.
 
 ```bash
-cargo test && cargo compiletest
+cargo test && cargo compiletest && cargo difftest
 ```
 
-## Adding Tests
+## Compile Tests
+
+### Adding Tests
 
 Rust-GPU's end-to-end test's use an external version of the [`compiletest`] tool
 as a testing framework. Be sure to check out the [repository][`compiletest`] and