Experiments: add Caliper mixin (#356)

Experiments may inherit Caliper, which can specify additional specs
that must be built to use Caliper when collecting metrics for an
benchmark.

Amg2023 is updated to make use of this: it inherits an option to
specify what type of metrics to collect with Caliper.

modifiers/caliper/modifier.py

@@ -6,6 +6,21 @@
 from ramble.modkit import *
 
 
+def add_mode(mode_name, mode_option, description):
+    mode(
+        name=mode_name,
+        description=description,
+    )
+
+    env_var_modification(
+        "CALI_CONFIG_MODE",
+        mode_option,
+        method="append",
+        separator=",",
+        modes=[mode_name],
+    )
+
+
 class Caliper(SpackModifier):
     """Define a modifier for Caliper"""
 
@@ -15,15 +30,57 @@ class Caliper(SpackModifier):
 
     maintainers("pearce8")
 
-    mode("time", description="Platform-independent collection of time")
-
     _cali_datafile = "{experiment_run_dir}/{experiment_name}.cali"
 
+    _default_mode = "time"
+
+    add_mode(
+        mode_name=_default_mode,
+        mode_option="time.exclusive",
+        description="Platform-independent collection of time (default mode)",
+    )
+
     env_var_modification(
         "CALI_CONFIG",
-        "spot(output={})".format(_cali_datafile),
+        "spot(output={}{})".format(_cali_datafile, "${CALI_CONFIG_MODE}"),
         method="set",
-        modes=["time"],
+        modes=[_default_mode],
+    )
+
+    add_mode(
+        mode_name="mpi",
+        mode_option="profile.mpi",
+        description="Profile MPI functions",
+    )
+
+    add_mode(
+        mode_name="cuda",
+        mode_option="profile.cuda",
+        description="Profile CUDA API functions",
+    )
+
+    add_mode(
+        mode_name="topdown-counters-all",
+        mode_option="topdown-counters.all",
+        description="Raw counter values for Intel top-down analysis (all levels)",
+    )
+
+    add_mode(
+        mode_name="topdown-counters-toplevel",
+        mode_option="topdown-counters.toplevel",
+        description="Raw counter values for Intel top-down analysis (top level)",
+    )
+
+    add_mode(
+        mode_name="topdown-all",
+        mode_option="topdown.all",
+        description="Top-down analysis for Intel CPUs (all levels)",
+    )
+
+    add_mode(
+        mode_name="topdown-toplevel",
+        mode_option="topdown.toplevel",
+        description="Top-down analysis for Intel CPUs (top level)",
     )
 
     archive_pattern(_cali_datafile)

var/exp_repo/experiments/amg2023/experiment.py

@@ -1,8 +1,9 @@
 from benchpark.directives import variant
 from benchpark.experiment import Experiment
+from benchpark.expr.builtin.caliper import Caliper
 
 
-class Amg2023(Experiment):
+class Amg2023(Caliper, Experiment):
     variant(
         "programming_model",
         default="openmp",
@@ -101,6 +102,11 @@ def make_experiment_example(self):
             }
         }
 
+    def compute_modifiers_section(self):
+        return Experiment.compute_modifiers_section(
+            self
+        ) + Caliper.compute_modifiers_section(self)
+
     def compute_applications_section(self):
         if self.spec.satisfies("workload=problem1"):
             self.workload = "problem1"
@@ -124,7 +130,6 @@ def compute_spack_section(self):
         # set package versions
         app_version = "develop"
         hypre_version = "2.31.0"
-        # caliper_version = "master"
 
         # get system config options
         # TODO: Get compiler/mpi/package handles directly from system.py
@@ -171,6 +176,14 @@ def compute_spack_section(self):
             "compiler": system_specs["compiler"],
         }
 
+        caliper_package_specs = Caliper.compute_spack_section(self)
+        if Caliper.is_enabled(self):
+            package_specs["hypre"]["pkg_spec"] += "+caliper"
+            package_specs[app_name]["pkg_spec"] += "+caliper"
+        else:
+            package_specs["hypre"]["pkg_spec"] += "~caliper"
+            package_specs[app_name]["pkg_spec"] += "~caliper"
+
         if self.spec.satisfies("programming_model=openmp"):
             package_specs["hypre"]["pkg_spec"] += "+openmp"
             package_specs[app_name]["pkg_spec"] += "+openmp"
@@ -190,6 +203,12 @@ def compute_spack_section(self):
             )
 
         return {
-            "packages": {k: v for k, v in package_specs.items() if v},
-            "environments": {app_name: {"packages": list(package_specs.keys())}},
+            "packages": {k: v for k, v in package_specs.items() if v}
+            | caliper_package_specs["packages"],
+            "environments": {
+                app_name: {
+                    "packages": list(package_specs.keys())
+                    + list(caliper_package_specs["packages"].keys())
+                }
+            },
         }

var/exp_repo/experiments/caliper/experiment.py

@@ -0,0 +1,93 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+
+from benchpark.directives import variant
+
+
+class Caliper:
+    variant(
+        "caliper",
+        default="none",
+        values=(
+            "none",
+            "time",
+            "mpi",
+            "cuda",
+            "topdown-counters-all",
+            "topdown-counters-toplevel",
+            "topdown-all",
+            "topdown-toplevel",
+        ),
+        multi=True,
+        description="caliper mode",
+    )
+
+    def is_enabled(self):
+        return not self.spec.satisfies("caliper=none")
+
+    def compute_modifiers_section(self):
+        modifier_list = []
+        if not self.spec.satisfies("caliper=none"):
+            for var in list(self.spec.variants["caliper"]):
+                if var != "time":
+                    caliper_modifier_modes = {}
+                    caliper_modifier_modes["name"] = "caliper"
+                    caliper_modifier_modes["mode"] = var
+                    modifier_list.append(caliper_modifier_modes)
+            # Add time as the last mode
+            modifier_list.append({"name": "caliper", "mode": "time"})
+        return modifier_list
+
+    def compute_spack_section(self):
+        # set package versions
+        caliper_version = "master"
+
+        # get system config options
+        # TODO: Get compiler/mpi/package handles directly from system.py
+        system_specs = {}
+        system_specs["compiler"] = "default-compiler"
+        if self.spec.satisfies("programming_model=cuda"):
+            system_specs["cuda_arch"] = "{cuda_arch}"
+        if self.spec.satisfies("programming_model=rocm"):
+            system_specs["rocm_arch"] = "{rocm_arch}"
+
+        # set package spack specs
+        package_specs = {}
+
+        if not self.spec.satisfies("caliper=none"):
+            package_specs["caliper"] = {
+                "pkg_spec": f"caliper@{caliper_version}+adiak+mpi~libunwind~libdw",
+                "compiler": system_specs["compiler"],
+            }
+            if any("topdown" in var for var in self.spec.variants["caliper"]):
+                papi_support = True  # check if target system supports papi
+                if papi_support:
+                    package_specs["caliper"]["pkg_spec"] += "+papi"
+                else:
+                    raise NotImplementedError(
+                        "Target system does not support the papi interface"
+                    )
+            elif self.spec.satisfies("caliper=cuda"):
+                cuda_support = (
+                    self.spec.satisfies("caliper=cuda") and True
+                )  # check if target system supports cuda
+                if cuda_support:
+                    package_specs["caliper"][
+                        "pkg_spec"
+                    ] += "~papi+cuda cuda_arch={}".format(system_specs["cuda_arch"])
+                else:
+                    raise NotImplementedError(
+                        "Target system does not support the cuda interface"
+                    )
+            elif self.spec.satisfies("caliper=time") or self.spec.satisfies(
+                "caliper=mpi"
+            ):
+                package_specs["caliper"]["pkg_spec"] += "~papi"
+
+        return {
+            "packages": {k: v for k, v in package_specs.items() if v},
+            "environments": {"caliper": {"packages": list(package_specs.keys())}},
+        }