diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index c1e8a2f76..0dd660f01 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -222,6 +222,32 @@ jobs: --disable-logger \ workspace setup --dry-run + - name: Dry run dynamic saxpy/openmp with dynamic Venado + run: | + ./bin/benchpark system init --dest=venado-system1 lanl-venado cuda=12.5 compiler=cce +gtl + system_id=$(./bin/benchpark system id ./venado-system1) + ./bin/benchpark experiment init --dest=saxpy-openmp-venado saxpy+openmp + ./bin/benchpark setup ./saxpy-openmp-venado ./venado-system1 workspace/ + . workspace/setup.sh + ramble \ + --workspace-dir "workspace/saxpy-openmp-venado/$system_id/workspace" \ + --disable-progress-bar \ + --disable-logger \ + workspace setup --dry-run + + - name: Dry run dynamic saxpy/cuda with dynamic Venado + run: | + ./bin/benchpark system init --dest=venado-system2 lanl-venado cuda=12.5 compiler=cce +gtl + system_id=$(./bin/benchpark system id ./venado-system2) + ./bin/benchpark experiment init --dest=saxpy-cuda-venado saxpy+cuda + ./bin/benchpark setup ./saxpy-cuda-venado ./venado-system2 workspace/ + . workspace/setup.sh + ramble \ + --workspace-dir "workspace/saxpy-cuda-venado/$system_id/workspace" \ + --disable-progress-bar \ + --disable-logger \ + workspace setup --dry-run + - name: Dry run dynamic gromacs/openmp with dynamic Ruby run: | ./bin/benchpark system init --dest=cluster-system llnl-cluster compiler=gcc diff --git a/legacy/systems/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml b/legacy/systems/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml index 1c050e729..2528cb68e 100644 --- a/legacy/systems/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml +++ b/legacy/systems/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml @@ -302,6 +302,7 @@ packages: gtl_cutoff_size: 4096 fi_cxi_ats: 0 gtl_lib_path: /opt/cray/pe/mpich/8.1.26/gtl/lib + gtl_libs: ["libmpi_gtl_hsa"] ldflags: "-L/opt/cray/pe/mpich/8.1.26/ofi/crayclang/16.0/lib -lmpi -L/opt/cray/pe/mpich/8.1.26/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/8.1.26/gtl/lib -lmpi_gtl_hsa" - spec: cray-mpich@8.1.26%cce@16.0.0 ~gtl +wrappers prefix: /opt/cray/pe/mpich/8.1.26/ofi/crayclang/16.0 diff --git a/repo/amg2023/package.py b/repo/amg2023/package.py index c7e2ac0c0..5cc54af78 100644 --- a/repo/amg2023/package.py +++ b/repo/amg2023/package.py @@ -36,7 +36,7 @@ class Amg2023(CMakePackage, CudaPackage, ROCmPackage): depends_on("hypre+cuda", when="+cuda") requires("+cuda", when="^hypre+cuda") - for arch in ("none", "50", "60", "70", "80"): + for arch in ("none", "50", "60", "70", "80", "90"): depends_on(f"hypre cuda_arch={arch}", when=f"cuda_arch={arch}") depends_on("hypre+rocm", when="+rocm") @@ -44,6 +44,10 @@ class Amg2023(CMakePackage, CudaPackage, ROCmPackage): for target in ("none", "gfx803", "gfx900", "gfx906", "gfx908", "gfx90a", "gfx942"): depends_on(f"hypre amdgpu_target={target}", when=f"amdgpu_target={target}") + def setup_build_environment(self, env): + if "+cuda" in self.spec: + env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") + def cmake_args(self): cmake_options = [] cmake_options.append(self.define_from_variant("AMG_WITH_CALIPER", "caliper")) diff --git a/repo/caliper/for_aarch64.patch b/repo/caliper/for_aarch64.patch new file mode 100644 index 000000000..d3fed9a09 --- /dev/null +++ b/repo/caliper/for_aarch64.patch @@ -0,0 +1,11 @@ +--- spack-src/src/services/callpath/Callpath.cpp.bak 2020-10-28 14:38:19.668122844 +0900 ++++ spack-src/src/services/callpath/Callpath.cpp 2020-10-28 15:03:12.258061188 +0900 +@@ -63,7 +63,7 @@ + unw_context_t unw_ctx; + unw_cursor_t unw_cursor; + +- unw_getcontext(&unw_ctx); ++ unw_getcontext(unw_ctx); + + if (unw_init_local(&unw_cursor, &unw_ctx) < 0) { + Log(0).stream() << "callpath: unable to init libunwind cursor" << endl; diff --git a/repo/caliper/package.py b/repo/caliper/package.py new file mode 100644 index 000000000..1b13781bd --- /dev/null +++ b/repo/caliper/package.py @@ -0,0 +1,291 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + +from spack.package import * + + +class Caliper(CachedCMakePackage, CudaPackage, ROCmPackage): + """Caliper is a program instrumentation and performance measurement + framework. It is designed as a performance analysis toolbox in a + library, allowing one to bake performance analysis capabilities + directly into applications and activate them at runtime. + """ + + homepage = "https://github.com/LLNL/Caliper" + git = "https://github.com/LLNL/Caliper.git" + url = "https://github.com/LLNL/Caliper/archive/v2.12.1.tar.gz" + tags = ["e4s", "radiuss"] + + maintainers("daboehme", "adrienbernede") + + test_requires_compiler = True + + license("BSD-3-Clause") + + version("master", branch="master") + version("2.12.1", sha256="2b5a8f98382c94dc75cc3f4517c758eaf9a3f9cea0a8dbdc7b38506060d6955c") + version("2.11.0", sha256="b86b733cbb73495d5f3fe06e6a9885ec77365c8aa9195e7654581180adc2217c") + version("2.10.0", sha256="14c4fb5edd5e67808d581523b4f8f05ace8549698c0e90d84b53171a77f58565") + version("2.9.1", sha256="4771d630de505eff9227e0ec498d0da33ae6f9c34df23cb201b56181b8759e9e") + version("2.9.0", sha256="507ea74be64a2dfd111b292c24c4f55f459257528ba51a5242313fa50978371f") + version( + "2.8.0", + sha256="17807b364b5ac4b05997ead41bd173e773f9a26ff573ff2fe61e0e70eab496e4", + deprecated=True, + ) + version( + "2.7.0", + sha256="b3bf290ec2692284c6b4f54cc0c507b5700c536571d3e1a66e56626618024b2b", + deprecated=True, + ) + version( + "2.6.0", + sha256="6efcd3e4845cc9a6169e0d934840766b12182c6d09aa3ceca4ae776e23b6360f", + deprecated=True, + ) + version( + "2.5.0", + sha256="d553e60697d61c53de369b9ca464eb30710bda90fba9671201543b64eeac943c", + deprecated=True, + ) + version( + "2.4.0", tag="v2.4.0", commit="30577b4b8beae104b2b35ed487fec52590a99b3d", deprecated=True + ) + version( + "2.3.0", tag="v2.3.0", commit="9fd89bb0120750d1f9dfe37bd963e24e478a2a20", deprecated=True + ) + version( + "2.2.0", tag="v2.2.0", commit="c408e9b3642c7aa80eff37b0826d819c57e7bc04", deprecated=True + ) + version( + "2.1.1", tag="v2.1.1", commit="0593b0e01c1d8d3e50c990399cc0fee403485599", deprecated=True + ) + version( + "2.0.1", tag="v2.0.1", commit="4d7ff46381c53a461e62edd949e2d9dea9db7b08", deprecated=True + ) + version( + "1.9.1", tag="v1.9.1", commit="cfc1defbbee20b50dd3e3477badd09a92b1df970", deprecated=True + ) + version( + "1.9.0", tag="v1.9.0", commit="8356e747349b285aa621c5b74e71559f0babc4a1", deprecated=True + ) + version( + "1.8.0", tag="v1.8.0", commit="117c1ef596b617dc71407b8b67eebef094a654f8", deprecated=True + ) + version( + "1.7.0", tag="v1.7.0", commit="898277c93d884d4e7ca1ffcf3bbea81d22364f26", deprecated=True + ) + + #depends_on("c", type="build") # generated + #depends_on("cxx", type="build") # generated + #depends_on("fortran", type="build") # generated + + is_linux = sys.platform.startswith("linux") + variant("shared", default=True, description="Build shared libraries") + variant("adiak", default=True, description="Enable Adiak support") + variant("mpi", default=True, description="Enable MPI support") + # libunwind has some issues on Mac + variant( + "libunwind", default=sys.platform != "darwin", description="Enable stack unwind support" + ) + variant("libdw", default=is_linux, description="Enable DWARF symbol lookup") + # pthread_self() signature is incompatible with PAPI_thread_init() on Mac + variant("papi", default=sys.platform != "darwin", description="Enable PAPI service") + variant("libpfm", default=False, description="Enable libpfm (perf_events) service") + # Gotcha is Linux-only + variant("gotcha", default=is_linux, description="Enable GOTCHA support") + variant("sampler", default=is_linux, description="Enable sampling support on Linux") + variant("sosflow", default=False, description="Enable SOSflow support") + variant("fortran", default=False, description="Enable Fortran support") + variant("variorum", default=False, description="Enable Variorum support") + variant("vtune", default=False, description="Enable Intel Vtune support") + variant("kokkos", default=True, when="@2.3.0:", description="Enable Kokkos profiling support") + variant("tests", default=False, description="Enable tests") + variant("tools", default=True, description="Enable tools") + variant("python", default=False, when="@v2.12:", description="Build Python bindings") + + depends_on("adiak@0.1:0", when="@2.2:2.10 +adiak") + depends_on("adiak@0.4:0", when="@2.11: +adiak") + + depends_on("papi@5.3:5", when="@:2.2 +papi") + depends_on("papi@5.3:", when="@2.3: +papi") + + depends_on("libpfm4@4.8:4", when="+libpfm") + + depends_on("mpi", when="+mpi") + depends_on("unwind@1.2:1", when="+libunwind") + depends_on("elfutils", when="+libdw") + depends_on("variorum", when="+variorum") + depends_on("intel-oneapi-vtune", when="+vtune") + + depends_on("sosflow@spack", when="@1.0:1+sosflow") + + depends_on("cmake", type="build") + depends_on("python", type="build") + + depends_on("python@3", when="+python", type=("build", "link", "run")) + depends_on("py-pybind11", when="+python", type=("build", "link", "run")) + + # sosflow support not yet in 2.0 + conflicts("+sosflow", "@2:") + conflicts("+adiak", "@:2.1") + conflicts("+libdw", "@:2.4") + conflicts("+rocm", "@:2.7") + conflicts("+rocm+cuda") + + patch("for_aarch64.patch", when="@:2.11 target=aarch64:") + patch( + "sampler-service-missing-libunwind-include-dir.patch", + when="@2.9.0:2.9.1 +libunwind +sampler", + ) + + def _get_sys_type(self, spec): + sys_type = spec.architecture + if "SYS_TYPE" in env: + sys_type = env["SYS_TYPE"] + return sys_type + + def initconfig_compiler_entries(self): + spec = self.spec + entries = super().initconfig_compiler_entries() + + if spec.satisfies("+rocm"): + entries.insert(0, cmake_cache_path("CMAKE_CXX_COMPILER", spec["hip"].hipcc)) + + entries.append(cmake_cache_option("WITH_FORTRAN", spec.satisfies("+fortran"))) + + entries.append(cmake_cache_option("BUILD_SHARED_LIBS", spec.satisfies("+shared"))) + entries.append(cmake_cache_option("BUILD_TESTING", spec.satisfies("+tests"))) + entries.append(cmake_cache_option("WITH_TOOLS", spec.satisfies("+tools"))) + entries.append(cmake_cache_option("BUILD_DOCS", False)) + entries.append(cmake_cache_path("PYTHON_EXECUTABLE", spec["python"].command.path)) + + return entries + + def initconfig_hardware_entries(self): + spec = self.spec + entries = super().initconfig_hardware_entries() + + if spec.satisfies("+cuda"): + entries.append(cmake_cache_option("WITH_CUPTI", True)) + entries.append(cmake_cache_option("WITH_NVTX", True)) + entries.append(cmake_cache_path("CUDA_TOOLKIT_ROOT_DIR", spec["cuda"].prefix)) + entries.append(cmake_cache_path("CUPTI_PREFIX", spec["cuda"].prefix)) + else: + entries.append(cmake_cache_option("WITH_CUPTI", False)) + entries.append(cmake_cache_option("WITH_NVTX", False)) + + if spec.satisfies("+rocm"): + entries.append(cmake_cache_option("WITH_ROCTRACER", True)) + entries.append(cmake_cache_option("WITH_ROCTX", True)) + else: + entries.append(cmake_cache_option("WITH_ROCTRACER", False)) + entries.append(cmake_cache_option("WITH_ROCTX", False)) + + return entries + + def initconfig_mpi_entries(self): + spec = self.spec + entries = super().initconfig_mpi_entries() + + entries.append(cmake_cache_option("WITH_MPI", spec.satisfies("+mpi"))) + + return entries + + def initconfig_package_entries(self): + spec = self.spec + entries = [] + + # TPL locations + entries.append("#------------------{0}".format("-" * 60)) + entries.append("# TPLs") + entries.append("#------------------{0}\n".format("-" * 60)) + + if spec.satisfies("+adiak"): + entries.append(cmake_cache_path("adiak_DIR", spec["adiak"].prefix)) + if spec.satisfies("+papi"): + entries.append(cmake_cache_path("PAPI_PREFIX", spec["papi"].prefix)) + if spec.satisfies("+libdw"): + entries.append(cmake_cache_path("LIBDW_PREFIX", spec["elfutils"].prefix)) + if spec.satisfies("+libpfm"): + entries.append(cmake_cache_path("LIBPFM_INSTALL", spec["libpfm4"].prefix)) + if spec.satisfies("+sosflow"): + entries.append(cmake_cache_path("SOS_PREFIX", spec["sosflow"].prefix)) + if spec.satisfies("+variorum"): + entries.append(cmake_cache_path("VARIORUM_PREFIX", spec["variorum"].prefix)) + if spec.satisfies("+vtune"): + itt_dir = join_path(spec["intel-oneapi-vtune"].prefix, "vtune", "latest") + entries.append(cmake_cache_path("ITT_PREFIX", itt_dir)) + if spec.satisfies("+libunwind"): + entries.append(cmake_cache_path("LIBUNWIND_PREFIX", spec["unwind"].prefix)) + + # Build options + entries.append("#------------------{0}".format("-" * 60)) + entries.append("# Build Options") + entries.append("#------------------{0}\n".format("-" * 60)) + + entries.append(cmake_cache_option("WITH_ADIAK", spec.satisfies("+adiak"))) + entries.append(cmake_cache_option("WITH_GOTCHA", spec.satisfies("+gotcha"))) + entries.append(cmake_cache_option("WITH_SAMPLER", spec.satisfies("+sampler"))) + entries.append(cmake_cache_option("WITH_PAPI", spec.satisfies("+papi"))) + entries.append(cmake_cache_option("WITH_LIBDW", spec.satisfies("+libdw"))) + entries.append(cmake_cache_option("WITH_LIBPFM", spec.satisfies("+libpfm"))) + entries.append(cmake_cache_option("WITH_SOSFLOW", spec.satisfies("+sosflow"))) + entries.append(cmake_cache_option("WITH_KOKKOS", spec.satisfies("+kokkos"))) + entries.append(cmake_cache_option("WITH_VARIORUM", spec.satisfies("+variorum"))) + entries.append(cmake_cache_option("WITH_VTUNE", spec.satisfies("+vtune"))) + entries.append(cmake_cache_option("WITH_PYTHON_BINDINGS", spec.satisfies("+python"))) + + # -DWITH_CALLPATH was renamed -DWITH_LIBUNWIND in 2.5 + callpath_flag = "LIBUNWIND" if spec.satisfies("@2.5:") else "CALLPATH" + entries.append(cmake_cache_option("WITH_%s" % callpath_flag, spec.satisfies("+libunwind"))) + + return entries + + def cmake_args(self): + return [] + + def setup_run_environment(self, env): + if self.spec.satisfies("+python"): + env.prepend_path("PYTHONPATH", self.spec.prefix.join(python_platlib)) + env.prepend_path("PYTHONPATH", self.spec.prefix.join(python_purelib)) + + @run_after("install") + def cache_test_sources(self): + """Copy the example source files after the package is installed to an + install test subdirectory for use during `spack test run`.""" + cache_extra_test_sources(self, [join_path("examples", "apps")]) + + def test_cxx_example(self): + """build and run cxx-example""" + + exe = "cxx-example" + source_file = "{0}.cpp".format(exe) + + source_path = find_required_file( + self.test_suite.current_test_cache_dir, source_file, expected=1, recursive=True + ) + + lib_dir = self.prefix.lib if os.path.exists(self.prefix.lib) else self.prefix.lib64 + + cxx = which(os.environ["CXX"]) + test_dir = os.path.dirname(source_path) + with working_dir(test_dir): + cxx( + "-L{0}".format(lib_dir), + "-I{0}".format(self.prefix.include), + source_path, + "-o", + exe, + "-std=c++11", + "-lcaliper", + "-lstdc++", + ) + + cxx_example = which(exe) + cxx_example() diff --git a/repo/caliper/sampler-service-missing-libunwind-include-dir.patch b/repo/caliper/sampler-service-missing-libunwind-include-dir.patch new file mode 100644 index 000000000..4acd660f6 --- /dev/null +++ b/repo/caliper/sampler-service-missing-libunwind-include-dir.patch @@ -0,0 +1,14 @@ +diff -ruN spack-src/src/services/sampler/CMakeLists.txt spack-src-patched/src/services/sampler/CMakeLists.txt +--- spack-src/src/services/sampler/CMakeLists.txt 2022-11-30 13:52:42.000000000 -0500 ++++ spack-src-patched/src/services/sampler/CMakeLists.txt 2023-05-04 20:43:47.240310306 -0400 +@@ -17,6 +17,10 @@ + + include_directories(${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + ++if (CALIPER_HAVE_LIBUNWIND) ++ include_directories(${LIBUNWIND_INCLUDE_DIRS}) ++endif() ++ + add_library(caliper-sampler OBJECT ${CALIPER_SAMPLER_SOURCES}) + + add_service_objlib("caliper-sampler") diff --git a/repo/cray-mpich/package.py b/repo/cray-mpich/package.py index b0f2e1561..36d80d64b 100644 --- a/repo/cray-mpich/package.py +++ b/repo/cray-mpich/package.py @@ -17,8 +17,8 @@ def libs(self): if self.spec.satisfies("+gtl"): gtl_lib_prefix = self.spec.extra_attributes["gtl_lib_path"] - libraries = ["libmpi_gtl_hsa"] - libs += find_libraries(libraries, root=gtl_lib_prefix, recursive=True) + gtl_libs = self.spec.extra_attributes["gtl_libs"] + libs += find_libraries(gtl_libs, root=gtl_lib_prefix, recursive=True) return libs @@ -29,8 +29,6 @@ def setup_run_environment(self, env): if self.spec.satisfies("+gtl"): env.set("MPICH_GPU_SUPPORT_ENABLED", "1") env.prepend_path("LD_LIBRARY_PATH", self.spec.extra_attributes["gtl_lib_path"]) - env.set("GTL_HSA_VSMSG_CUTOFF_SIZE", str(self.spec.extra_attributes["gtl_cutoff_size"])) - env.set("FI_CXI_ATS", str(self.spec.extra_attributes["fi_cxi_ats"])) else: env.set("MPICH_GPU_SUPPORT_ENABLED", "0") gtl_path = self.spec.extra_attributes.get("gtl_lib_path", "") diff --git a/repo/hypre/package.py b/repo/hypre/package.py index 31613bcd3..8a12e268d 100644 --- a/repo/hypre/package.py +++ b/repo/hypre/package.py @@ -54,3 +54,5 @@ def setup_build_environment(self, env): if spec.satisfies("+openmp"): libsci_name += "_mp" env.append_flags("LDFLAGS", f"-L{spec['lapack'].prefix}/lib -l{libsci_name}") + if "+cuda" in self.spec: + env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") diff --git a/repo/saxpy/CMakeLists.txt b/repo/saxpy/CMakeLists.txt index fc0c5945b..9369aa4d2 100644 --- a/repo/saxpy/CMakeLists.txt +++ b/repo/saxpy/CMakeLists.txt @@ -43,6 +43,7 @@ if(USE_CUDA) set_source_files_properties(saxpy.cc PROPERTIES LANGUAGE "CUDA") set(CMAKE_CUDA_STANDARD_REQUIRED TRUE) set(CMAKE_CXX_STANDARD_REQUIRED TRUE) + set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_CUDA_FLAGS "-Xcompiler -DUSE_CUDA") set_target_properties(${PROJECT_NAME} PROPERTIES CUDA_SEPARABLE_COMPILATION ON) endif() diff --git a/repo/saxpy/package.py b/repo/saxpy/package.py index 56f0c211e..25dc76ce0 100644 --- a/repo/saxpy/package.py +++ b/repo/saxpy/package.py @@ -42,6 +42,7 @@ def stage_source(self): def setup_build_environment(self, env): if "+cuda" in self.spec: env.set("CUDAHOSTCXX", self.spec["mpi"].mpicxx) + env.set("NVCC_APPEND_FLAGS", "-std=c++14 -allow-unsupported-compiler") def cmake_args(self): spec = self.spec diff --git a/systems/all_system_definitions/HPECray-neoverse-H100-Slingshot/system_definition.yaml b/systems/all_system_definitions/HPECray-neoverse-H100-Slingshot/system_definition.yaml new file mode 100644 index 000000000..982de3828 --- /dev/null +++ b/systems/all_system_definitions/HPECray-neoverse-H100-Slingshot/system_definition.yaml @@ -0,0 +1,32 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +system_definition: + name: HPECray-neoverse-H100-Slingshot + integrator: + vendor: HPECray + name: EX254n + processor: + vendor: NVIDIA + name: Grace + ISA: Armv9 + uArch: neoverse + accelerator: + vendor: NVIDIA + name: H100 + ISA: PTX + uArch: sm_90 + interconnect: + vendor: HPECray + name: Slingshot11 + systems-tested: + lanl-venado: + os: HPECrayOS + scheduler: slurm + compiler: cce + runtime: cuda + mpi: cray-mpich + top500-system-instances: + - Venado (LANL) diff --git a/systems/lanl-venado/compilers/cce/00-cce-18-compilers.yaml b/systems/lanl-venado/compilers/cce/00-cce-18-compilers.yaml new file mode 100644 index 000000000..7ce4aeb35 --- /dev/null +++ b/systems/lanl-venado/compilers/cce/00-cce-18-compilers.yaml @@ -0,0 +1,25 @@ +compilers: +- compiler: + spec: cce@18.0.0 + paths: + cc: /opt/cray/pe/cce/18.0.0/bin/craycc + cxx: /opt/cray/pe/cce/18.0.0/bin/crayCC + f77: /opt/cray/pe/cce/18.0.0/bin/crayftn + fc: /opt/cray/pe/cce/18.0.0/bin/crayftn + flags: + cflags: -g -O2 --gcc-toolchain=/usr/projects/hpcsoft/tce/24-07/cos3-aarch64-cc90/compilers/gcc/12.3.0 + cxxflags: -g -O2 --gcc-toolchain=/usr/projects/hpcsoft/tce/24-07/cos3-aarch64-cc90/compilers/gcc/12.3.0 + fflags: -g -O2 -hnopattern + ldflags: -ldl + operating_system: sles15 + target: aarch64 + modules: [] + environment: + prepend_path: + LD_LIBRARY_PATH: "/opt/cray/pe/cce/18.0.0/cce/aarch64/lib:/opt/cray/libfabric/1.20.1/lib64:/usr/projects/hpcsoft/tce/24-07/cos3-aarch64-cc90/compilers/gcc/12.3.0/lib:/usr/projects/hpcsoft/tce/24-07/cos3-aarch64-cc90/compilers/gcc/12.3.0/lib64:/opt/cray/pe/gcc-libs" + extra_rpaths: + - /opt/cray/pe/gcc-libs + - /opt/cray/pe/cce/18.0.0/cce/aarch64/lib + - /opt/cray/libfabric/1.20.1/lib64 + - /usr/projects/hpcsoft/tce/24-07/cos3-aarch64-cc90/compilers/gcc/12.3.0/lib + - /usr/projects/hpcsoft/tce/24-07/cos3-aarch64-cc90/compilers/gcc/12.3.0/lib64 diff --git a/systems/lanl-venado/compilers/gcc/00-gcc-12-compilers.yaml b/systems/lanl-venado/compilers/gcc/00-gcc-12-compilers.yaml new file mode 100644 index 000000000..f28815e8f --- /dev/null +++ b/systems/lanl-venado/compilers/gcc/00-gcc-12-compilers.yaml @@ -0,0 +1,14 @@ +compilers: +- compiler: + spec: gcc@12.3.0 + paths: + cc: /usr/projects/hpcsoft/tce/24-07/cos3-aarch64-cc90/compilers/gcc/12.3.0/bin/gcc + cxx: /usr/projects/hpcsoft/tce/24-07/cos3-aarch64-cc90/compilers/gcc/12.3.0/bin/g++ + f77: /usr/projects/hpcsoft/tce/24-07/cos3-aarch64-cc90/compilers/gcc/12.3.0/bin/gfortran + fc: /usr/projects/hpcsoft/tce/24-07/cos3-aarch64-cc90/compilers/gcc/12.3.0/bin/gfortran + flags: {} + operating_system: sles15 + target: aarch64 + modules: [] + environment: {} + extra_rpaths: [] diff --git a/systems/lanl-venado/externals/base/00-packages.yaml b/systems/lanl-venado/externals/base/00-packages.yaml new file mode 100644 index 000000000..63682360b --- /dev/null +++ b/systems/lanl-venado/externals/base/00-packages.yaml @@ -0,0 +1,43 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +packages: + tar: + externals: + - spec: tar@1.34 + prefix: /usr + buildable: false + cmake: + externals: + - spec: cmake@3.29.6 + prefix: /usr/projects/hpcsoft/tce/24-07/cos3-aarch64-cc90/packages/cmake/cmake-3.29.6 + buildable: false + gmake: + externals: + - spec: gmake@4.2.1 + prefix: /usr + buildable: false + automake: + externals: + - spec: automake@1.15.1 + prefix: /usr + buildable: false + autoconf: + externals: + - spec: autoconf@2.69 + prefix: /usr + buildable: false + fftw: + externals: + - spec: fftw@3.3.10.8 + prefix: /opt/cray/pe/fftw/3.3.10.8/arm_grace + buildable: false + python: + externals: + - spec: python@3.10.9 + prefix: /usr/projects/hpcsoft/common/aarch64/anaconda/2023.03-python-3.10 + buildable: false + mpi: + buildable: false diff --git a/systems/lanl-venado/externals/libsci/00-gcc-packages.yaml b/systems/lanl-venado/externals/libsci/00-gcc-packages.yaml new file mode 100644 index 000000000..f58d5a59a --- /dev/null +++ b/systems/lanl-venado/externals/libsci/00-gcc-packages.yaml @@ -0,0 +1,5 @@ +packages: + cray-libsci: + externals: + - spec: cray-libsci@24.07.0%gcc + prefix: /opt/cray/pe/libsci/24.07.0/gnu/12.3/aarch64 diff --git a/systems/lanl-venado/externals/libsci/01-cce-packages.yaml b/systems/lanl-venado/externals/libsci/01-cce-packages.yaml new file mode 100644 index 000000000..8c1608b6b --- /dev/null +++ b/systems/lanl-venado/externals/libsci/01-cce-packages.yaml @@ -0,0 +1,5 @@ +packages: + cray-libsci: + externals: + - spec: cray-libsci@24.07.0%cce + prefix: /opt/cray/pe/libsci/24.07.0/cray/17.0/aarch64 diff --git a/systems/lanl-venado/system.py b/systems/lanl-venado/system.py new file mode 100644 index 000000000..67e7d2038 --- /dev/null +++ b/systems/lanl-venado/system.py @@ -0,0 +1,220 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +import pathlib + +from benchpark.directives import variant +from benchpark.system import System + +id_to_resources = { + "grace-hopper": { + "sys_cores_per_node": 144, + "sys_gpus_per_node": 4, + }, + "grace-grace": { + "sys_cores_per_node": 144, + }, +} + + +class LanlVenado(System): + variant( + "cluster", + default="grace-hopper", + values=("grace-hopper", "grace-grace"), + description="Which cluster to run on", + ) + + variant( + "cuda", + default="12-5", + values=("11.8", "12.5"), + description="CUDA version", + ) + + variant( + "compiler", + default="cce", + values=("gcc", "cce"), + description="Which compiler to use", + ) + + variant( + "gtl", + default=False, + values=(True, False), + description="Use GTL-enabled MPI", + ) + + variant( + "lapack", + default="cusolver", + values=("cusolver", "cray-libsci"), + description="Which lapack to use", + ) + + variant( + "blas", + default="cublas", + values=("cublas", "cray-libsci"), + description="Which blas to use", + ) + + def initialize(self): + super().initialize() + + self.scheduler = "slurm" + attrs = id_to_resources.get(self.spec.variants["cluster"][0]) + for k, v in attrs.items(): + setattr(self, k, v) + + def generate_description(self, output_dir): + super().generate_description(output_dir) + + sw_description = pathlib.Path(output_dir) / "software.yaml" + + with open(sw_description, "w") as f: + f.write(self.sw_description()) + + def system_specific_variables(self): + return { + "cuda_arch": "90", + "default_cuda_version": self.spec.variants["cuda"][0], + "extra_batch_opts": '"-A llnl_ai_g -pgpu"', + } + + def external_pkg_configs(self): + externals = LanlVenado.resource_location / "externals" + + selections = [externals / "base" / "00-packages.yaml"] + + cuda_cfg_path = self.next_adhoc_cfg() + with open(cuda_cfg_path, "w") as f: + f.write(self.cuda_config(self.spec.variants["cuda"][0])) + selections.append(cuda_cfg_path) + + mpi_cfg_path = self.next_adhoc_cfg() + with open(mpi_cfg_path, "w") as f: + f.write(self.mpi_config()) + selections.append(mpi_cfg_path) + + if self.spec.satisfies("compiler=cce"): + selections.append(externals / "libsci" / "01-cce-packages.yaml") + elif self.spec.satisfies("compiler=gcc"): + selections.append(externals / "libsci" / "00-gcc-packages.yaml") + + return selections + + def compiler_configs(self): + compilers = LanlVenado.resource_location / "compilers" + + selections = [] + # TODO: Construct/extract/customize compiler information from the working set + if self.spec.satisfies("compiler=cce"): + selections.append(compilers / "cce" / "00-cce-18-compilers.yaml") + selections.append(compilers / "gcc" / "00-gcc-12-compilers.yaml") + + return selections + + def mpi_config(self): + mpi_version = "8.1.30" + gtl = ( + "+gtl" + if self.spec.satisfies("compiler=cce") and self.spec.satisfies("+gtl") + else "~gtl" + ) + + # TODO: Construct/extract this information from the working set + if self.spec.satisfies("compiler=cce"): + compiler = "cce@18.0.0" + mpi_compiler_suffix = "crayclang/17.0" + elif self.spec.satisfies("compiler=gcc"): + compiler = "gcc@12.3.0" + mpi_compiler_suffix = "gnu/12.3" + + return f"""\ +packages: + cray-mpich: + externals: + - spec: cray-mpich@{mpi_version}%{compiler} {gtl} +wrappers + prefix: /opt/cray/pe/mpich/{mpi_version}/ofi/{mpi_compiler_suffix} + extra_attributes: + gtl_lib_path: /opt/cray/pe/mpich/{mpi_version}/gtl/lib + gtl_libs: ["libmpi_gtl_cuda"] + ldflags: "-L/opt/cray/pe/mpich/{mpi_version}/ofi/{mpi_compiler_suffix}/lib -lmpi -L/opt/cray/pe/mpich/{mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{mpi_version}/gtl/lib -lmpi_gtl_cuda" +""" + + def cuda_config(self, cuda_version): + template = """\ +packages: + blas: + require: + - {blas} + lapack: + require: + - {lapack} + curand: + externals: + - spec: curand@{x} + prefix: /opt/nvidia/hpc_sdk/Linux_aarch64/24.7/cuda/{x} + buildable: false + cusparse: + externals: + - spec: cusparse@{x} + prefix: /opt/nvidia/hpc_sdk/Linux_aarch64/24.7/cuda/{x} + buildable: false + cuda: + externals: + - spec: cuda@{x} + prefix: /opt/nvidia/hpc_sdk/Linux_aarch64/24.7/cuda/{x} + buildable: false + cub: + externals: + - spec: cub@{x} + prefix: /opt/nvidia/hpc_sdk/Linux_aarch64/24.7/cuda/{x} + buildable: false + cublas: + externals: + - spec: cublas@{x} + prefix: /opt/nvidia/hpc_sdk/Linux_aarch64/24.7/math_libs/{x} + buildable: false + cusolver: + externals: + - spec: cusolver@{x} + prefix: /opt/nvidia/hpc_sdk/Linux_aarch64/24.7/math_libs/{x} + buildable: false + cufft: + externals: + - spec: cufft@{x} + prefix: /opt/nvidia/hpc_sdk/Linux_aarch64/24.7/math_libs/{x} + buildable: false +""" + return template.format( + x=cuda_version, + blas=self.spec.variants["blas"][0], + lapack=self.spec.variants["lapack"][0], + ) + + def sw_description(self): + """This is somewhat vestigial: for the Tioga config that is committed + to the repo, multiple instances of mpi/compilers are stored and + and these variables were used to choose consistent dependencies. + The configs generated by this class should only ever have one + instance of MPI etc., so there is no need for that. The experiments + will fail if these variables are not defined though, so for now + they are still generated (but with more-generic values). + """ + return f"""\ +software: + packages: + default-compiler: + pkg_spec: {self.spec.variants["compiler"][0]} + default-mpi: + pkg_spec: cray-mpich + default-lapack: + pkg_spec: {self.spec.variants["lapack"][0]} + default-blas: + pkg_spec: {self.spec.variants["blas"][0]} +""" diff --git a/systems/llnl-elcapitan/system.py b/systems/llnl-elcapitan/system.py index 1f4b4fb30..335e39390 100644 --- a/systems/llnl-elcapitan/system.py +++ b/systems/llnl-elcapitan/system.py @@ -163,6 +163,7 @@ def mpi_config(self, cce_version): gtl_cutoff_size: 4096 fi_cxi_ats: 0 gtl_lib_path: /opt/cray/pe/mpich/{mpi_version}/gtl/lib + gtl_libs: ["libmpi_gtl_hsa"] ldflags: "-L/opt/cray/pe/mpich/{mpi_version}/ofi/crayclang/{short_cce_version}/lib -lmpi -L/opt/cray/pe/mpich/{mpi_version}/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/{mpi_version}/gtl/lib -lmpi_gtl_hsa" """