venado system specs (#548)

* venado system specs

* Fix NVCC flags

* Fix caliper version

* Add patches

* Fix caliper, slurm issues

* Fix flag

* Add gcc toolchain

* lint, license

* lint

* Remove redundant code

* amg2023 venado build

* Fix gtl lib names

* remove one_of clause

* el cap compiler dependency

* Create system_definition.yaml

* cufft

* Add dry runs

---------

Co-authored-by: Riyaz Haque <[email protected]>
Co-authored-by: pearce8 <[email protected]>

Author: 3 people

.github/workflows/run.yml

@@ -222,6 +222,32 @@ jobs:
             --disable-logger \
             workspace setup --dry-run
 
+      - name: Dry run dynamic saxpy/openmp with dynamic Venado
+        run: |
+          ./bin/benchpark system init --dest=venado-system1 lanl-venado cuda=12.5 compiler=cce +gtl
+          system_id=$(./bin/benchpark system id ./venado-system1)
+          ./bin/benchpark experiment init --dest=saxpy-openmp-venado saxpy+openmp
+          ./bin/benchpark setup ./saxpy-openmp-venado ./venado-system1 workspace/
+          . workspace/setup.sh
+          ramble \
+            --workspace-dir "workspace/saxpy-openmp-venado/$system_id/workspace" \
+            --disable-progress-bar \
+            --disable-logger \
+            workspace setup --dry-run
+
+      - name: Dry run dynamic saxpy/cuda with dynamic Venado
+        run: |
+          ./bin/benchpark system init --dest=venado-system2 lanl-venado cuda=12.5 compiler=cce +gtl
+          system_id=$(./bin/benchpark system id ./venado-system2)
+          ./bin/benchpark experiment init --dest=saxpy-cuda-venado saxpy+cuda
+          ./bin/benchpark setup ./saxpy-cuda-venado ./venado-system2 workspace/
+          . workspace/setup.sh
+          ramble \
+            --workspace-dir "workspace/saxpy-cuda-venado/$system_id/workspace" \
+            --disable-progress-bar \
+            --disable-logger \
+            workspace setup --dry-run
+
       - name: Dry run dynamic gromacs/openmp with dynamic Ruby
         run: |
           ./bin/benchpark system init --dest=cluster-system llnl-cluster compiler=gcc 

legacy/systems/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml

@@ -302,6 +302,7 @@ packages:
         gtl_cutoff_size: 4096
         fi_cxi_ats: 0
         gtl_lib_path: /opt/cray/pe/mpich/8.1.26/gtl/lib
+        gtl_libs: ["libmpi_gtl_hsa"]
         ldflags: "-L/opt/cray/pe/mpich/8.1.26/ofi/crayclang/16.0/lib -lmpi -L/opt/cray/pe/mpich/8.1.26/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/8.1.26/gtl/lib -lmpi_gtl_hsa"
     - spec: [email protected]%[email protected] ~gtl +wrappers
       prefix: /opt/cray/pe/mpich/8.1.26/ofi/crayclang/16.0

repo/amg2023/package.py

@@ -36,14 +36,18 @@ class Amg2023(CMakePackage, CudaPackage, ROCmPackage):
 
     depends_on("hypre+cuda", when="+cuda")
     requires("+cuda", when="^hypre+cuda")
-    for arch in ("none", "50", "60", "70", "80"):
+    for arch in ("none", "50", "60", "70", "80", "90"):
         depends_on(f"hypre cuda_arch={arch}", when=f"cuda_arch={arch}")
 
     depends_on("hypre+rocm", when="+rocm")
     requires("+rocm", when="^hypre+rocm")
     for target in ("none", "gfx803", "gfx900", "gfx906", "gfx908", "gfx90a", "gfx942"):
         depends_on(f"hypre amdgpu_target={target}", when=f"amdgpu_target={target}")
 
+    def setup_build_environment(self, env):
+        if "+cuda" in self.spec:
+            env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler")
+
     def cmake_args(self):
         cmake_options = []
         cmake_options.append(self.define_from_variant("AMG_WITH_CALIPER", "caliper"))

repo/caliper/for_aarch64.patch

@@ -0,0 +1,11 @@
+--- spack-src/src/services/callpath/Callpath.cpp.bak	2020-10-28 14:38:19.668122844 +0900
++++ spack-src/src/services/callpath/Callpath.cpp	2020-10-28 15:03:12.258061188 +0900
+@@ -63,7 +63,7 @@
+         unw_context_t unw_ctx;
+         unw_cursor_t  unw_cursor;
+ 
+-        unw_getcontext(&unw_ctx);
++        unw_getcontext(unw_ctx);
+ 
+         if (unw_init_local(&unw_cursor, &unw_ctx) < 0) {
+             Log(0).stream() << "callpath: unable to init libunwind cursor" << endl;

repo/caliper/package.py

@@ -0,0 +1,291 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import sys
+
+from spack.package import *
+
+
+class Caliper(CachedCMakePackage, CudaPackage, ROCmPackage):
+    """Caliper is a program instrumentation and performance measurement
+    framework. It is designed as a performance analysis toolbox in a
+    library, allowing one to bake performance analysis capabilities
+    directly into applications and activate them at runtime.
+    """
+
+    homepage = "https://github.com/LLNL/Caliper"
+    git = "https://github.com/LLNL/Caliper.git"
+    url = "https://github.com/LLNL/Caliper/archive/v2.12.1.tar.gz"
+    tags = ["e4s", "radiuss"]
+
+    maintainers("daboehme", "adrienbernede")
+
+    test_requires_compiler = True
+
+    license("BSD-3-Clause")
+
+    version("master", branch="master")
+    version("2.12.1", sha256="2b5a8f98382c94dc75cc3f4517c758eaf9a3f9cea0a8dbdc7b38506060d6955c")
+    version("2.11.0", sha256="b86b733cbb73495d5f3fe06e6a9885ec77365c8aa9195e7654581180adc2217c")
+    version("2.10.0", sha256="14c4fb5edd5e67808d581523b4f8f05ace8549698c0e90d84b53171a77f58565")
+    version("2.9.1", sha256="4771d630de505eff9227e0ec498d0da33ae6f9c34df23cb201b56181b8759e9e")
+    version("2.9.0", sha256="507ea74be64a2dfd111b292c24c4f55f459257528ba51a5242313fa50978371f")
+    version(
+        "2.8.0",
+        sha256="17807b364b5ac4b05997ead41bd173e773f9a26ff573ff2fe61e0e70eab496e4",
+        deprecated=True,
+    )
+    version(
+        "2.7.0",
+        sha256="b3bf290ec2692284c6b4f54cc0c507b5700c536571d3e1a66e56626618024b2b",
+        deprecated=True,
+    )
+    version(
+        "2.6.0",
+        sha256="6efcd3e4845cc9a6169e0d934840766b12182c6d09aa3ceca4ae776e23b6360f",
+        deprecated=True,
+    )
+    version(
+        "2.5.0",
+        sha256="d553e60697d61c53de369b9ca464eb30710bda90fba9671201543b64eeac943c",
+        deprecated=True,
+    )
+    version(
+        "2.4.0", tag="v2.4.0", commit="30577b4b8beae104b2b35ed487fec52590a99b3d", deprecated=True
+    )
+    version(
+        "2.3.0", tag="v2.3.0", commit="9fd89bb0120750d1f9dfe37bd963e24e478a2a20", deprecated=True
+    )
+    version(
+        "2.2.0", tag="v2.2.0", commit="c408e9b3642c7aa80eff37b0826d819c57e7bc04", deprecated=True
+    )
+    version(
+        "2.1.1", tag="v2.1.1", commit="0593b0e01c1d8d3e50c990399cc0fee403485599", deprecated=True
+    )
+    version(
+        "2.0.1", tag="v2.0.1", commit="4d7ff46381c53a461e62edd949e2d9dea9db7b08", deprecated=True
+    )
+    version(
+        "1.9.1", tag="v1.9.1", commit="cfc1defbbee20b50dd3e3477badd09a92b1df970", deprecated=True
+    )
+    version(
+        "1.9.0", tag="v1.9.0", commit="8356e747349b285aa621c5b74e71559f0babc4a1", deprecated=True
+    )
+    version(
+        "1.8.0", tag="v1.8.0", commit="117c1ef596b617dc71407b8b67eebef094a654f8", deprecated=True
+    )
+    version(
+        "1.7.0", tag="v1.7.0", commit="898277c93d884d4e7ca1ffcf3bbea81d22364f26", deprecated=True
+    )
+
+    #depends_on("c", type="build")  # generated
+    #depends_on("cxx", type="build")  # generated
+    #depends_on("fortran", type="build")  # generated
+
+    is_linux = sys.platform.startswith("linux")
+    variant("shared", default=True, description="Build shared libraries")
+    variant("adiak", default=True, description="Enable Adiak support")
+    variant("mpi", default=True, description="Enable MPI support")
+    # libunwind has some issues on Mac
+    variant(
+        "libunwind", default=sys.platform != "darwin", description="Enable stack unwind support"
+    )
+    variant("libdw", default=is_linux, description="Enable DWARF symbol lookup")
+    # pthread_self() signature is incompatible with PAPI_thread_init() on Mac
+    variant("papi", default=sys.platform != "darwin", description="Enable PAPI service")
+    variant("libpfm", default=False, description="Enable libpfm (perf_events) service")
+    # Gotcha is Linux-only
+    variant("gotcha", default=is_linux, description="Enable GOTCHA support")
+    variant("sampler", default=is_linux, description="Enable sampling support on Linux")
+    variant("sosflow", default=False, description="Enable SOSflow support")
+    variant("fortran", default=False, description="Enable Fortran support")
+    variant("variorum", default=False, description="Enable Variorum support")
+    variant("vtune", default=False, description="Enable Intel Vtune support")
+    variant("kokkos", default=True, when="@2.3.0:", description="Enable Kokkos profiling support")
+    variant("tests", default=False, description="Enable tests")
+    variant("tools", default=True, description="Enable tools")
+    variant("python", default=False, when="@v2.12:", description="Build Python bindings")
+
+    depends_on("[email protected]:0", when="@2.2:2.10 +adiak")
+    depends_on("[email protected]:0", when="@2.11: +adiak")
+
+    depends_on("[email protected]:5", when="@:2.2 +papi")
+    depends_on("[email protected]:", when="@2.3: +papi")
+
+    depends_on("[email protected]:4", when="+libpfm")
+
+    depends_on("mpi", when="+mpi")
+    depends_on("[email protected]:1", when="+libunwind")
+    depends_on("elfutils", when="+libdw")
+    depends_on("variorum", when="+variorum")
+    depends_on("intel-oneapi-vtune", when="+vtune")
+
+    depends_on("sosflow@spack", when="@1.0:1+sosflow")
+
+    depends_on("cmake", type="build")
+    depends_on("python", type="build")
+
+    depends_on("python@3", when="+python", type=("build", "link", "run"))
+    depends_on("py-pybind11", when="+python", type=("build", "link", "run"))
+
+    # sosflow support not yet in 2.0
+    conflicts("+sosflow", "@2:")
+    conflicts("+adiak", "@:2.1")
+    conflicts("+libdw", "@:2.4")
+    conflicts("+rocm", "@:2.7")
+    conflicts("+rocm+cuda")
+
+    patch("for_aarch64.patch", when="@:2.11 target=aarch64:")
+    patch(
+        "sampler-service-missing-libunwind-include-dir.patch",
+        when="@2.9.0:2.9.1 +libunwind +sampler",
+    )
+
+    def _get_sys_type(self, spec):
+        sys_type = spec.architecture
+        if "SYS_TYPE" in env:
+            sys_type = env["SYS_TYPE"]
+        return sys_type
+
+    def initconfig_compiler_entries(self):
+        spec = self.spec
+        entries = super().initconfig_compiler_entries()
+
+        if spec.satisfies("+rocm"):
+            entries.insert(0, cmake_cache_path("CMAKE_CXX_COMPILER", spec["hip"].hipcc))
+
+        entries.append(cmake_cache_option("WITH_FORTRAN", spec.satisfies("+fortran")))
+
+        entries.append(cmake_cache_option("BUILD_SHARED_LIBS", spec.satisfies("+shared")))
+        entries.append(cmake_cache_option("BUILD_TESTING", spec.satisfies("+tests")))
+        entries.append(cmake_cache_option("WITH_TOOLS", spec.satisfies("+tools")))
+        entries.append(cmake_cache_option("BUILD_DOCS", False))
+        entries.append(cmake_cache_path("PYTHON_EXECUTABLE", spec["python"].command.path))
+
+        return entries
+
+    def initconfig_hardware_entries(self):
+        spec = self.spec
+        entries = super().initconfig_hardware_entries()
+
+        if spec.satisfies("+cuda"):
+            entries.append(cmake_cache_option("WITH_CUPTI", True))
+            entries.append(cmake_cache_option("WITH_NVTX", True))
+            entries.append(cmake_cache_path("CUDA_TOOLKIT_ROOT_DIR", spec["cuda"].prefix))
+            entries.append(cmake_cache_path("CUPTI_PREFIX", spec["cuda"].prefix))
+        else:
+            entries.append(cmake_cache_option("WITH_CUPTI", False))
+            entries.append(cmake_cache_option("WITH_NVTX", False))
+
+        if spec.satisfies("+rocm"):
+            entries.append(cmake_cache_option("WITH_ROCTRACER", True))
+            entries.append(cmake_cache_option("WITH_ROCTX", True))
+        else:
+            entries.append(cmake_cache_option("WITH_ROCTRACER", False))
+            entries.append(cmake_cache_option("WITH_ROCTX", False))
+
+        return entries
+
+    def initconfig_mpi_entries(self):
+        spec = self.spec
+        entries = super().initconfig_mpi_entries()
+
+        entries.append(cmake_cache_option("WITH_MPI", spec.satisfies("+mpi")))
+
+        return entries
+
+    def initconfig_package_entries(self):
+        spec = self.spec
+        entries = []
+
+        # TPL locations
+        entries.append("#------------------{0}".format("-" * 60))
+        entries.append("# TPLs")
+        entries.append("#------------------{0}\n".format("-" * 60))
+
+        if spec.satisfies("+adiak"):
+            entries.append(cmake_cache_path("adiak_DIR", spec["adiak"].prefix))
+        if spec.satisfies("+papi"):
+            entries.append(cmake_cache_path("PAPI_PREFIX", spec["papi"].prefix))
+        if spec.satisfies("+libdw"):
+            entries.append(cmake_cache_path("LIBDW_PREFIX", spec["elfutils"].prefix))
+        if spec.satisfies("+libpfm"):
+            entries.append(cmake_cache_path("LIBPFM_INSTALL", spec["libpfm4"].prefix))
+        if spec.satisfies("+sosflow"):
+            entries.append(cmake_cache_path("SOS_PREFIX", spec["sosflow"].prefix))
+        if spec.satisfies("+variorum"):
+            entries.append(cmake_cache_path("VARIORUM_PREFIX", spec["variorum"].prefix))
+        if spec.satisfies("+vtune"):
+            itt_dir = join_path(spec["intel-oneapi-vtune"].prefix, "vtune", "latest")
+            entries.append(cmake_cache_path("ITT_PREFIX", itt_dir))
+        if spec.satisfies("+libunwind"):
+            entries.append(cmake_cache_path("LIBUNWIND_PREFIX", spec["unwind"].prefix))
+
+        # Build options
+        entries.append("#------------------{0}".format("-" * 60))
+        entries.append("# Build Options")
+        entries.append("#------------------{0}\n".format("-" * 60))
+
+        entries.append(cmake_cache_option("WITH_ADIAK", spec.satisfies("+adiak")))
+        entries.append(cmake_cache_option("WITH_GOTCHA", spec.satisfies("+gotcha")))
+        entries.append(cmake_cache_option("WITH_SAMPLER", spec.satisfies("+sampler")))
+        entries.append(cmake_cache_option("WITH_PAPI", spec.satisfies("+papi")))
+        entries.append(cmake_cache_option("WITH_LIBDW", spec.satisfies("+libdw")))
+        entries.append(cmake_cache_option("WITH_LIBPFM", spec.satisfies("+libpfm")))
+        entries.append(cmake_cache_option("WITH_SOSFLOW", spec.satisfies("+sosflow")))
+        entries.append(cmake_cache_option("WITH_KOKKOS", spec.satisfies("+kokkos")))
+        entries.append(cmake_cache_option("WITH_VARIORUM", spec.satisfies("+variorum")))
+        entries.append(cmake_cache_option("WITH_VTUNE", spec.satisfies("+vtune")))
+        entries.append(cmake_cache_option("WITH_PYTHON_BINDINGS", spec.satisfies("+python")))
+
+        # -DWITH_CALLPATH was renamed -DWITH_LIBUNWIND in 2.5
+        callpath_flag = "LIBUNWIND" if spec.satisfies("@2.5:") else "CALLPATH"
+        entries.append(cmake_cache_option("WITH_%s" % callpath_flag, spec.satisfies("+libunwind")))
+
+        return entries
+
+    def cmake_args(self):
+        return []
+
+    def setup_run_environment(self, env):
+        if self.spec.satisfies("+python"):
+            env.prepend_path("PYTHONPATH", self.spec.prefix.join(python_platlib))
+            env.prepend_path("PYTHONPATH", self.spec.prefix.join(python_purelib))
+
+    @run_after("install")
+    def cache_test_sources(self):
+        """Copy the example source files after the package is installed to an
+        install test subdirectory for use during `spack test run`."""
+        cache_extra_test_sources(self, [join_path("examples", "apps")])
+
+    def test_cxx_example(self):
+        """build and run cxx-example"""
+
+        exe = "cxx-example"
+        source_file = "{0}.cpp".format(exe)
+
+        source_path = find_required_file(
+            self.test_suite.current_test_cache_dir, source_file, expected=1, recursive=True
+        )
+
+        lib_dir = self.prefix.lib if os.path.exists(self.prefix.lib) else self.prefix.lib64
+
+        cxx = which(os.environ["CXX"])
+        test_dir = os.path.dirname(source_path)
+        with working_dir(test_dir):
+            cxx(
+                "-L{0}".format(lib_dir),
+                "-I{0}".format(self.prefix.include),
+                source_path,
+                "-o",
+                exe,
+                "-std=c++11",
+                "-lcaliper",
+                "-lstdc++",
+            )
+
+            cxx_example = which(exe)
+            cxx_example()

repo/caliper/sampler-service-missing-libunwind-include-dir.patch

@@ -0,0 +1,14 @@
+diff -ruN spack-src/src/services/sampler/CMakeLists.txt spack-src-patched/src/services/sampler/CMakeLists.txt
+--- spack-src/src/services/sampler/CMakeLists.txt	2022-11-30 13:52:42.000000000 -0500
++++ spack-src-patched/src/services/sampler/CMakeLists.txt	2023-05-04 20:43:47.240310306 -0400
+@@ -17,6 +17,10 @@
+ 
+ include_directories(${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR})
+ 
++if (CALIPER_HAVE_LIBUNWIND)
++	  include_directories(${LIBUNWIND_INCLUDE_DIRS})
++endif()
++
+ add_library(caliper-sampler OBJECT ${CALIPER_SAMPLER_SOURCES})
+ 
+ add_service_objlib("caliper-sampler")

repo/cray-mpich/package.py

@@ -17,8 +17,8 @@ def libs(self):
 
         if self.spec.satisfies("+gtl"):
             gtl_lib_prefix = self.spec.extra_attributes["gtl_lib_path"]
-            libraries = ["libmpi_gtl_hsa"]
-            libs += find_libraries(libraries, root=gtl_lib_prefix, recursive=True)
+            gtl_libs = self.spec.extra_attributes["gtl_libs"]
+            libs += find_libraries(gtl_libs, root=gtl_lib_prefix, recursive=True)
 
         return libs
 
@@ -29,8 +29,6 @@ def setup_run_environment(self, env):
         if self.spec.satisfies("+gtl"):
             env.set("MPICH_GPU_SUPPORT_ENABLED", "1")
             env.prepend_path("LD_LIBRARY_PATH", self.spec.extra_attributes["gtl_lib_path"])
-            env.set("GTL_HSA_VSMSG_CUTOFF_SIZE", str(self.spec.extra_attributes["gtl_cutoff_size"]))
-            env.set("FI_CXI_ATS", str(self.spec.extra_attributes["fi_cxi_ats"]))
         else:
             env.set("MPICH_GPU_SUPPORT_ENABLED", "0")
             gtl_path = self.spec.extra_attributes.get("gtl_lib_path", "")