feat: initial support for Clang HIP (#2045)

Signed-off-by: Gavin Zhao <[email protected]>

.github/workflows/integration-tests.yml

@@ -575,6 +575,78 @@ jobs:
 
           ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]"
 
+  hip:
+    # Probably wouldn't matter anyway since we run in a container, but staying 
+    # close to the version is better than not.
+    runs-on: ubuntu-22.04
+    needs: build
+    container:
+      image: rocm/dev-ubuntu-22.04:6.0
+
+    env:
+      # SCCACHE_GHA_ENABLED: "on"
+      ROCM_PATH: "/opt/rocm"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      # I don't want to break the cache during testing. Will turn on after I
+      # make sure it's working.
+      #
+      # - name: Configure Cache Env
+      #   uses: actions/github-script@v7
+      #   with:
+      #     script: |
+      #       core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
+      #       core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '')
+
+      # - name: Configure ROCm Env
+      #   uses: actions/github-script@v7
+      #   with:
+      #     script: |
+      #       core.exportVariable('ROCM_PATH', process.env.ROCM_PATH || '');
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: integration-tests
+          path: /home/runner/.cargo/bin/
+      - name: Chmod for binary
+        run: chmod +x ${SCCACHE_PATH}
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          ## Install dependencies
+          sudo apt-get update
+          sudo apt-get install -y cmake
+
+      # Ensure that HIPCC isn't already borken
+      - name: Sanity Check
+        run: |
+          hipcc -o vectoradd_hip --offload-arch=gfx900 tests/cmake-hip/vectoradd_hip.cpp
+
+      - name: Test
+        run: |
+          cmake -B build -S tests/cmake-hip -DCMAKE_HIP_COMPILER_LAUNCHER=${SCCACHE_PATH} -DCMAKE_HIP_ARCHITECTURES=gfx900
+          cmake --build build
+
+      - name: Output
+        run: |
+          ${SCCACHE_PATH} --show-stats
+
+      - name: Test Twice for Cache Read
+        run: |
+          rm -rf build
+          cmake -B build -S tests/cmake-hip -DCMAKE_HIP_COMPILER_LAUNCHER=${SCCACHE_PATH} -DCMAKE_HIP_ARCHITECTURES=gfx900
+          cmake --build build
+
+      - name: Output
+        run: |
+          ${SCCACHE_PATH} --show-stats
+
+          ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]"
+
+
   gcc:
     runs-on: ubuntu-latest
     needs: build

src/compiler/c.rs

@@ -228,6 +228,55 @@ where
             compiler,
         })
     }
+
+    fn extract_rocm_arg(args: &ParsedArguments, flag: &str) -> Option<PathBuf> {
+        args.common_args.iter().find_map(|arg| match arg.to_str() {
+            Some(sarg) if sarg.starts_with(flag) => {
+                Some(PathBuf::from(sarg[arg.len()..].to_string()))
+            }
+            _ => None,
+        })
+    }
+
+    fn extract_rocm_env(env_vars: &[(OsString, OsString)], name: &str) -> Option<PathBuf> {
+        env_vars.iter().find_map(|(k, v)| match v.to_str() {
+            Some(path) if k == name => Some(PathBuf::from(path.to_string())),
+            _ => None,
+        })
+    }
+
+    // See https://clang.llvm.org/docs/HIPSupport.html for details regarding the
+    // order in which the environment variables and command-line arguments control the
+    // directory to search for bitcode libraries.
+    fn search_hip_device_libs(
+        args: &ParsedArguments,
+        env_vars: &[(OsString, OsString)],
+    ) -> Vec<PathBuf> {
+        let rocm_path_arg: Option<PathBuf> = Self::extract_rocm_arg(args, "--rocm-path=");
+        let hip_device_lib_path_arg: Option<PathBuf> =
+            Self::extract_rocm_arg(args, "--hip-device-lib-path=");
+        let rocm_path_env: Option<PathBuf> = Self::extract_rocm_env(env_vars, "ROCM_PATH");
+        let hip_device_lib_path_env: Option<PathBuf> =
+            Self::extract_rocm_env(env_vars, "HIP_DEVICE_LIB_PATH");
+
+        let hip_device_lib_path: PathBuf = hip_device_lib_path_arg
+            .or(hip_device_lib_path_env)
+            .or(rocm_path_arg.map(|path| path.join("amdgcn").join("bitcode")))
+            .or(rocm_path_env.map(|path| path.join("amdgcn").join("bitcode")))
+            // This is the default location in official AMD packages and containers.
+            .unwrap_or(PathBuf::from("/opt/rocm/amdgcn/bitcode"));
+
+        hip_device_lib_path
+            .read_dir()
+            .ok()
+            .map(|f| {
+                f.flatten()
+                    .filter(|f| f.path().extension().map_or(false, |ext| ext == "bc"))
+                    .map(|f| f.path())
+                    .collect()
+            })
+            .unwrap_or(Vec::default())
+    }
 }
 
 impl<T: CommandCreatorSync, I: CCompilerImpl> Compiler<T> for CCompiler<I> {
@@ -249,11 +298,29 @@ impl<T: CommandCreatorSync, I: CCompilerImpl> Compiler<T> for CCompiler<I> {
     ) -> CompilerArguments<Box<dyn CompilerHasher<T> + 'static>> {
         match self.compiler.parse_arguments(arguments, cwd) {
             CompilerArguments::Ok(mut args) => {
+                // Handle SCCACHE_EXTRAFILES
                 for (k, v) in env_vars.iter() {
                     if k.as_os_str() == OsStr::new("SCCACHE_EXTRAFILES") {
                         args.extra_hash_files.extend(std::env::split_paths(&v))
                     }
                 }
+
+                // Handle cache invalidation for the ROCm device bitcode libraries. Every HIP
+                // object links in some LLVM bitcode libraries (.bc files), so in some sense
+                // every HIP object compilation has an direct dependency on those bitcode
+                // libraries.
+                //
+                // The bitcode libraries are unlikely to change **except** when a ROCm version
+                // changes, so for correctness we should take these bitcode libraries into
+                // account by adding them to `extra_hash_files`.
+                //
+                // In reality, not every available bitcode library is needed, but that is
+                // too much to handle on our side so we just hash every bitcode library we find.
+                if args.language == Language::Hip {
+                    args.extra_hash_files
+                        .extend(Self::search_hip_device_libs(&args, env_vars))
+                }
+
                 CompilerArguments::Ok(Box::new(CCompilerHasher {
                     parsed_args: args,
                     executable: self.executable.clone(),
@@ -1490,6 +1557,7 @@ mod test {
         t("mm", Language::ObjectiveCxx);
 
         t("cu", Language::Cuda);
+        t("hip", Language::Hip);
     }
 
     #[test]

src/compiler/clang.rs

@@ -167,6 +167,9 @@ impl CCompilerImpl for Clang {
 
 counted_array!(pub static ARGS: [ArgInfo<gcc::ArgData>; _] = [
     take_arg!("--dependent-lib", OsString, Concatenated('='), PassThrough),
+    take_arg!("--hip-device-lib-path", PathBuf, Concatenated('='), PassThroughPath),
+    take_arg!("--hip-path", PathBuf, Concatenated('='), PassThroughPath),
+    take_arg!("--rocm-path", PathBuf, Concatenated('='), PassThroughPath),
     take_arg!("--serialize-diagnostics", OsString, Separated, PassThrough),
     take_arg!("--target", OsString, Separated, PassThrough),
     // Note: for clang we must override the dep options from gcc.rs with `CanBeSeparated`.
@@ -181,6 +184,7 @@ counted_array!(pub static ARGS: [ArgInfo<gcc::ArgData>; _] = [
     flag!("-fcolor-diagnostics", DiagnosticsColorFlag),
     flag!("-fcxx-modules", TooHardFlag),
     take_arg!("-fdebug-compilation-dir", OsString, Separated, PassThrough),
+    take_arg!("-fembed-offload-object", PathBuf, Concatenated('='), ExtraHashFile),
     flag!("-fmodules", TooHardFlag),
     flag!("-fno-color-diagnostics", NoDiagnosticsColorFlag),
     flag!("-fno-pch-timestamp", PassThroughFlag),
@@ -415,6 +419,138 @@ mod test {
         );
     }
 
+    #[test]
+    fn test_parse_arguments_hip() {
+        let a = parses!("-c", "foo.hip", "-o", "foo.o");
+        assert_eq!(Some("foo.hip"), a.input.to_str());
+        assert_eq!(Language::Hip, a.language);
+        assert_map_contains!(
+            a.outputs,
+            (
+                "obj",
+                ArtifactDescriptor {
+                    path: PathBuf::from("foo.o"),
+                    optional: false
+                }
+            )
+        );
+        assert!(a.preprocessor_args.is_empty());
+        assert!(a.common_args.is_empty());
+    }
+
+    #[test]
+    fn test_parse_arguments_hip_flags() {
+        let a = parses!(
+            "-c",
+            "foo.cpp",
+            "-x",
+            "hip",
+            "--offload-arch=gfx900",
+            "-o",
+            "foo.o"
+        );
+        assert_eq!(Some("foo.cpp"), a.input.to_str());
+        assert_eq!(Language::Hip, a.language);
+        assert_map_contains!(
+            a.outputs,
+            (
+                "obj",
+                ArtifactDescriptor {
+                    path: PathBuf::from("foo.o"),
+                    optional: false
+                }
+            )
+        );
+        assert!(a.preprocessor_args.is_empty());
+        assert_eq!(ovec!["--offload-arch=gfx900"], a.common_args);
+
+        let b = parses!(
+            "-c",
+            "foo.cpp",
+            "-x",
+            "hip",
+            "--offload-arch=gfx900",
+            "-o",
+            "foo.o"
+        );
+        assert_eq!(Some("foo.cpp"), b.input.to_str());
+        assert_eq!(Language::Hip, b.language);
+        assert_map_contains!(
+            b.outputs,
+            (
+                "obj",
+                ArtifactDescriptor {
+                    path: PathBuf::from("foo.o"),
+                    optional: false
+                }
+            )
+        );
+        assert!(b.preprocessor_args.is_empty());
+        assert_eq!(ovec!["--offload-arch=gfx900"], b.common_args);
+    }
+
+    #[test]
+    fn test_parse_arguments_hip_paths() {
+        let a = parses!(
+            "-c",
+            "foo.cpp",
+            "-x",
+            "hip",
+            "--offload-arch=gfx900",
+            "-o",
+            "foo.o",
+            "--hip-path=/usr"
+        );
+        assert_eq!(Some("foo.cpp"), a.input.to_str());
+        assert_eq!(Language::Hip, a.language);
+        assert_map_contains!(
+            a.outputs,
+            (
+                "obj",
+                ArtifactDescriptor {
+                    path: PathBuf::from("foo.o"),
+                    optional: false
+                }
+            )
+        );
+        assert!(a.preprocessor_args.is_empty());
+        assert_eq!(
+            ovec!["--offload-arch=gfx900", "--hip-path=/usr"],
+            a.common_args
+        );
+
+        let b = parses!(
+            "-c",
+            "foo.cpp",
+            "-x",
+            "hip",
+            "--offload-arch=gfx900",
+            "-o",
+            "foo.o",
+            "--hip-device-lib-path=/usr/lib64/amdgcn/bitcode"
+        );
+        assert_eq!(Some("foo.cpp"), b.input.to_str());
+        assert_eq!(Language::Hip, b.language);
+        assert_map_contains!(
+            b.outputs,
+            (
+                "obj",
+                ArtifactDescriptor {
+                    path: PathBuf::from("foo.o"),
+                    optional: false
+                }
+            )
+        );
+        assert!(b.preprocessor_args.is_empty());
+        assert_eq!(
+            ovec![
+                "--offload-arch=gfx900",
+                "--hip-device-lib-path=/usr/lib64/amdgcn/bitcode"
+            ],
+            b.common_args
+        );
+    }
+
     #[test]
     fn test_dependent_lib() {
         let a = parses!(

src/compiler/compiler.rs

@@ -113,6 +113,7 @@ pub enum Language {
     ObjectiveCxx,
     Cuda,
     Rust,
+    Hip,
 }
 
 impl Language {
@@ -135,6 +136,7 @@ impl Language {
             Some("cu") => Some(Language::Cuda),
             // TODO cy
             Some("rs") => Some(Language::Rust),
+            Some("hip") => Some(Language::Hip),
             e => {
                 trace!("Unknown source extension: {}", e.unwrap_or("(None)"));
                 None
@@ -151,6 +153,7 @@ impl Language {
             Language::ObjectiveCxx => "objc++",
             Language::Cuda => "cuda",
             Language::Rust => "rust",
+            Language::Hip => "hip",
         }
     }
 }
@@ -167,6 +170,7 @@ impl CompilerKind {
             | Language::ObjectiveCxx => "C/C++",
             Language::Cuda => "CUDA",
             Language::Rust => "Rust",
+            Language::Hip => "HIP",
         }
         .to_string()
     }

src/compiler/gcc.rs

@@ -381,6 +381,7 @@ where
                     "cu" => Some(Language::Cuda),
                     "rs" => Some(Language::Rust),
                     "cuda" => Some(Language::Cuda),
+                    "hip" => Some(Language::Hip),
                     _ => cannot_cache!("-x"),
                 };
             }
@@ -643,7 +644,8 @@ fn language_to_gcc_arg(lang: Language) -> Option<&'static str> {
         Language::ObjectiveC => Some("objective-c"),
         Language::ObjectiveCxx => Some("objective-c++"),
         Language::Cuda => Some("cu"),
-        Language::Rust => None,          // Let the compiler decide
+        Language::Rust => None, // Let the compiler decide
+        Language::Hip => Some("hip"),
         Language::GenericHeader => None, // Let the compiler decide
     }
 }

tests/cmake-hip/CMakeLists.txt

@@ -0,0 +1,6 @@
+cmake_minimum_required(VERSION 3.10)
+
+project(myproject LANGUAGES CXX HIP)
+
+add_library(vectoradd_hip vectoradd_hip.cpp)
+set_source_files_properties(vectoradd_hip.cpp PROPERTIES LANGUAGE HIP)