Move system command logic; use unique ID in generated dirs (#360)

* move 'benchpark setup' to its own cmd module; refactor some resources into separate modules that are used by setup and main

* system_id includes hash of spec string, so system configs generated from different specs have unique ramble workspaces when generated by 'benchpark setup'

* 'benchpark system init --basedir' now uses same unique identifier as system_id.yaml to generate dirname

* Trigger CI

* changes in lib/ were not triggering ci

* need to relocate benchpark_systems

* style fixes

* ramble workspace generated by 'benchpark setup' needs to account for system hash in test run

* excess imports; remove redefinition

.github/workflows/ci.yml

@@ -43,6 +43,7 @@ jobs:
               - 'var/**'
               - 'README.rst'
               - 'tags.yaml'
+              - 'lib/**'
             style:
               - '.github/**'
               - 'bin/**'
@@ -51,6 +52,7 @@ jobs:
               - 'experiments/**'
               - 'repo/**'
               - 'var/**'
+              - 'lib/**'
             run:
               - '.github/**'
               - 'bin/**'
@@ -59,6 +61,7 @@ jobs:
               - 'experiments/**'
               - 'repo/**'
               - 'var/**'
+              - 'lib/**'
             license:
               - '.github/**'
               - 'bin/**'
@@ -67,6 +70,7 @@ jobs:
               - 'experiments/**'
               - 'repo/**'
               - 'var/**'
+              - 'lib/**'
 
   docs:
     if: ${{ needs.changes.outputs.docs == 'true' }}

.github/workflows/run.yml

@@ -156,7 +156,7 @@ jobs:
           ./bin/benchpark setup kripke/rocm ./tioga-system workspace/
           . workspace/setup.sh
           ramble \
-            --workspace-dir workspace/kripke/rocm/Tioga/workspace \
+            --workspace-dir workspace/kripke/rocm/Tioga-d34a754/workspace \
             --disable-progress-bar \
             --disable-logger \
             workspace setup --dry-run

lib/benchpark/accounting.py

@@ -0,0 +1,37 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+
+from benchpark.paths import source_location
+
+
+def benchpark_experiments():
+    source_dir = source_location()
+    experiments = []
+    experiments_dir = source_dir / "experiments"
+    for x in os.listdir(experiments_dir):
+        for y in os.listdir(experiments_dir / x):
+            experiments.append(f"{x}/{y}")
+    return experiments
+
+
+def benchpark_modifiers():
+    source_dir = source_location()
+    modifiers = []
+    for x in os.listdir(source_dir / "modifiers"):
+        modifiers.append(x)
+    return modifiers
+
+
+def benchpark_systems():
+    source_dir = source_location()
+    systems = []
+    for x in os.listdir(source_dir / "configs"):
+        if not (
+            os.path.isfile(os.path.join(source_dir / "configs", x)) or x == "common"
+        ):
+            systems.append(x)
+    return systems

lib/benchpark/cmd/setup.py

@@ -0,0 +1,244 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import pathlib
+import shutil
+import sys
+import yaml
+
+from benchpark.accounting import (
+    benchpark_experiments,
+    benchpark_modifiers,
+    benchpark_systems,
+)
+from benchpark.debug import debug_print
+from benchpark.paths import source_location
+from benchpark.runtime import RuntimeResources
+
+
+# Note: it would be nice to vendor spack.llnl.util.link_tree, but that
+# involves pulling in most of llnl/util/ and spack/util/
+def symlink_tree(src, dst, include_fn=None):
+    """Like ``cp -R`` but instead of files, create symlinks"""
+    src = os.path.abspath(src)
+    dst = os.path.abspath(dst)
+    # By default, we include all filenames
+    include_fn = include_fn or (lambda f: True)
+    for x in [src, dst]:
+        if not os.path.isdir(x):
+            raise ValueError(f"Not a directory: {x}")
+    for src_subdir, directories, files in os.walk(src):
+        relative_src_dir = pathlib.Path(os.path.relpath(src_subdir, src))
+        dst_dir = pathlib.Path(dst) / relative_src_dir
+        dst_dir.mkdir(parents=True, exist_ok=True)
+        for x in files:
+            if not include_fn(x):
+                continue
+            dst_symlink = dst_dir / x
+            src_file = os.path.join(src_subdir, x)
+            os.symlink(src_file, dst_symlink)
+
+
+def setup_parser(root_parser):
+    root_parser.add_argument(
+        "experiment",
+        type=str,
+        help="The experiment (benchmark/ProgrammingModel) to run",
+    )
+    root_parser.add_argument(
+        "system", type=str, help="The system on which to run the experiment"
+    )
+    root_parser.add_argument(
+        "experiments_root",
+        type=str,
+        help="Where to install packages and store results for the experiments. Benchpark expects to manage this directory, and it should be empty/nonexistent the first time you run benchpark setup experiments.",
+    )
+    root_parser.add_argument(
+        "--modifier",
+        type=str,
+        default="none",
+        help="The modifier to apply to the experiment (default none)",
+    )
+
+
+def benchpark_check_experiment(arg_str):
+    experiments = benchpark_experiments()
+    found = arg_str in experiments
+    if not found:
+        out_str = f'Invalid experiment (benchmark/ProgrammingModel) "{arg_str}" - must choose one of: '
+        for experiment in experiments:
+            out_str += f"\n\t{experiment}"
+        raise ValueError(out_str)
+    return found
+
+
+def benchpark_check_system(arg_str):
+    # First check if it's a directory that contains a system_id.yaml
+    cfg_path = pathlib.Path(arg_str)
+    if cfg_path.is_dir():
+        system_id_path = cfg_path / "system_id.yaml"
+        if system_id_path.exists():
+            with open(system_id_path, "r") as f:
+                data = yaml.safe_load(f)
+            name = data["system"]["name"]
+            spec_hash = data["system"]["config-hash"]
+            system_id = f"{name}-{spec_hash[:7]}"
+            return system_id, cfg_path
+
+    # If it's not a directory, it might be a shorthand that refers
+    # to a pre-constructed config
+    systems = benchpark_systems()
+    if arg_str not in systems:
+        out_str = (
+            f"Invalid system {arg_str}: must choose one of:"
+            "\n\t(a) A system ID from `benchpark systems`"
+            "\n\t(b) A directory containing system_id.yaml"
+        )
+        raise ValueError(out_str)
+
+    configs_src_dir = source_location() / "configs" / str(arg_str)
+    return arg_str, configs_src_dir
+
+
+def benchpark_check_modifier(arg_str):
+    modifiers = benchpark_modifiers()
+    found = arg_str in modifiers
+    if not found:
+        out_str = f'Invalid modifier "{arg_str}" - must choose one of: '
+        for modifier in modifiers:
+            out_str += f"\n\t{modifier}"
+        raise ValueError(out_str)
+    return found
+
+
+def command(args):
+    """
+    experiments_root/
+        spack/
+        ramble/
+        <experiment>/
+            <system>/
+                workspace/
+                    configs/
+                        (everything from source/configs/<system>)
+                        (everything from source/experiments/<experiment>)
+    """
+
+    experiment = args.experiment
+    system = args.system
+    experiments_root = pathlib.Path(os.path.abspath(args.experiments_root))
+    modifier = args.modifier
+    source_dir = source_location()
+    debug_print(f"source_dir = {source_dir}")
+    debug_print(f"specified experiment (benchmark/ProgrammingModel) = {experiment}")
+    benchpark_check_experiment(experiment)
+    debug_print(f"specified system = {system}")
+    system_id, configs_src_dir = benchpark_check_system(system)
+    debug_print(f"specified modifier = {modifier}")
+    benchpark_check_modifier(modifier)
+
+    workspace_dir = experiments_root / str(experiment) / str(system_id)
+
+    if workspace_dir.exists():
+        if workspace_dir.is_dir():
+            print(f"Clearing existing workspace {workspace_dir}")
+            shutil.rmtree(workspace_dir)
+        else:
+            print(
+                f"Benchpark expects to manage {workspace_dir} as a directory, but it is not"
+            )
+            sys.exit(1)
+
+    workspace_dir.mkdir(parents=True)
+
+    ramble_workspace_dir = workspace_dir / "workspace"
+    ramble_configs_dir = ramble_workspace_dir / "configs"
+    ramble_logs_dir = ramble_workspace_dir / "logs"
+    ramble_spack_experiment_configs_dir = (
+        ramble_configs_dir / "auxiliary_software_files"
+    )
+
+    print(f"Setting up configs for Ramble workspace {ramble_configs_dir}")
+
+    experiment_src_dir = source_dir / "experiments" / experiment
+    modifier_config_dir = source_dir / "modifiers" / modifier / "configs"
+    ramble_configs_dir.mkdir(parents=True)
+    ramble_logs_dir.mkdir(parents=True)
+    ramble_spack_experiment_configs_dir.mkdir(parents=True)
+
+    def include_fn(fname):
+        # Only include .yaml files
+        # Always exclude files that start with "."
+        if fname.startswith("."):
+            return False
+        if fname.endswith(".yaml"):
+            return True
+        return False
+
+    symlink_tree(configs_src_dir, ramble_configs_dir, include_fn)
+    symlink_tree(experiment_src_dir, ramble_configs_dir, include_fn)
+    symlink_tree(modifier_config_dir, ramble_configs_dir, include_fn)
+    symlink_tree(
+        source_dir / "configs" / "common",
+        ramble_spack_experiment_configs_dir,
+        include_fn,
+    )
+
+    template_name = "execute_experiment.tpl"
+    experiment_template_options = [
+        configs_src_dir / template_name,
+        experiment_src_dir / template_name,
+        source_dir / "common-resources" / template_name,
+    ]
+    for choice_template in experiment_template_options:
+        if os.path.exists(choice_template):
+            break
+    os.symlink(
+        choice_template,
+        ramble_configs_dir / "execute_experiment.tpl",
+    )
+
+    initializer_script = experiments_root / "setup.sh"
+
+    per_workspace_setup = RuntimeResources(experiments_root)
+
+    spack, first_time_spack = per_workspace_setup.spack_first_time_setup()
+    ramble, first_time_ramble = per_workspace_setup.ramble_first_time_setup()
+
+    if first_time_spack:
+        spack("repo", "add", "--scope=site", f"{source_dir}/repo")
+
+    if first_time_ramble:
+        ramble(f"repo add --scope=site {source_dir}/repo")
+        ramble('config --scope=site add "config:disable_progress_bar:true"')
+        ramble(f"repo add -t modifiers --scope=site {source_dir}/modifiers")
+        ramble("config --scope=site add \"config:spack:global:args:'-d'\"")
+
+    if not initializer_script.exists():
+        with open(initializer_script, "w") as f:
+            f.write(
+                f"""\
+if [ -n "${{_BENCHPARK_INITIALIZED:-}}" ]; then
+    return 0
+fi
+
+. {per_workspace_setup.spack_location}/share/spack/setup-env.sh
+. {per_workspace_setup.ramble_location}/share/ramble/setup-env.sh
+
+export SPACK_DISABLE_LOCAL_CONFIG=1
+
+export _BENCHPARK_INITIALIZED=true
+"""
+            )
+
+    instructions = f"""\
+To complete the benchpark setup, do the following:
+
+    . {initializer_script}
+
+Further steps are needed to build the experiments (ramble -P -D {ramble_workspace_dir} workspace setup) and run them (ramble -P -D {ramble_workspace_dir} on)
+"""
+    print(instructions)

lib/benchpark/cmd/system.py

@@ -20,7 +20,7 @@ def system_init(args):
 
     if args.basedir:
         base = args.basedir
-        sysdir = system.system_id()
+        sysdir = system.system_uid()
         destdir = os.path.join(base, sysdir)
     elif args.dest:
         destdir = args.dest

lib/benchpark/debug.py

@@ -0,0 +1,11 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+DEBUG = False
+
+
+def debug_print(message):
+    if DEBUG:
+        print("(debug) " + str(message))

lib/benchpark/paths.py

@@ -1,3 +1,6 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
 # SPDX-License-Identifier: Apache-2.0
 
 import os
@@ -6,3 +9,8 @@
 benchpark_home = pathlib.Path(os.path.expanduser("~/.benchpark"))
 global_ramble_path = benchpark_home / "ramble"
 global_spack_path = benchpark_home / "spack"
+
+
+def source_location():
+    the_directory_with_this_file = os.path.dirname(os.path.abspath(__file__))
+    return pathlib.Path(the_directory_with_this_file).parent.parent

lib/benchpark/system.py

@@ -96,17 +96,21 @@ def generate_description(self, output_dir):
         self.external_packages(output_dir)
         self.compiler_description(output_dir)
 
+        spec_hash = self.system_uid()
+
         system_id_path = output_dir / "system_id.yaml"
         with open(system_id_path, "w") as f:
             f.write(
                 f"""\
 system:
   name: {self.__class__.__name__}
+  spec: {str(self.spec)}
+  config-hash: {spec_hash}
 """
             )
 
-    def system_id(self):
-        return _hash_id([self.variables_yaml()])
+    def system_uid(self):
+        return _hash_id([str(self.spec)])
 
     def _merge_config_files(self, schema, selections, dst_path):
         data = cfg.read_config_file(selections[0], schema)