From f645c499c6e741168b095c253169f6d9aad5aeef Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 20 Aug 2024 12:17:24 -0700 Subject: [PATCH] Add command: `benchpark system init` (#298) * Create Spec object with subclasses ExperimentSpec and SystemSpec Spec objects have corrosponding ConcreteSpecObjects, which are immutable. Spec objects use the same syntax as Spack specs for describing variants of systems and experiments that can be instantiated to a directory * Create Experiment and System objects Experiment and System objects can be parametrized by variants Experiment and System objects can be described by the corrosponding specs These objects contain lifecycle methods that create the yaml files ingested by ramble Experiment and System objects live in Repos from `ramble.repository.Repo` * Bootstrap Ramble and Spack Bootstrap download Ramble and Spack to ~/.benchpark directory to import Ramble imports are executed normally Spack imports have to be manually imported by path to avoid conflict with Ramble internals * Refactor benchpark commands Benchpark commands defined in `benchpark.main` from lib directory * `benchpark-python` option * Implement System and Experiment examples Systems for Tioga and AWS with variants Experiment for Saxpy with variants * `benchpark system init` command Takes a SystemSpec as input Concretizes to a ConcreteSystemSpec Generates a System associated with the ConcreteSystemSpec Creates all yaml files associated with that System These yaml files can be ingested as a system to `benchpark setup` --------- Co-authored-by: Gregory Becker Co-authored-by: Alec Scott --- bin/benchpark | 538 +---------------- bin/benchpark-python | 21 + lib/benchpark/__init__.py | 0 lib/benchpark/cmd/system.py | 68 +++ lib/benchpark/directives.py | 202 +++++++ lib/benchpark/error.py | 14 + lib/benchpark/experiment.py | 98 ++++ lib/benchpark/paths.py | 8 + lib/benchpark/repo.py | 201 +++++++ lib/benchpark/runtime.py | 147 +++++ lib/benchpark/spec.py | 548 ++++++++++++++++++ lib/benchpark/system.py | 172 ++++++ lib/benchpark/variant.py | 118 ++++ lib/main.py | 527 +++++++++++++++++ pyproject.toml | 28 +- var/exp_repo/experiments/saxpy/experiment.py | 68 +++ var/exp_repo/repo.yaml | 2 + var/sys_repo/repo.yaml | 2 + var/sys_repo/systems/aws/system.py | 37 ++ .../compilers/gcc/00-gcc-12-compilers.yaml | 14 + .../compilers/rocm/00-rocm-551-compilers.yaml | 49 ++ .../compilers/rocm/01-rocm-543-compilers.yaml | 26 + .../tioga/externals/base/00-packages.yaml | 186 ++++++ .../externals/libsci/00-gcc-packages.yaml | 5 + .../externals/libsci/01-cce-packages.yaml | 5 + .../externals/mpi/00-gcc-ngtl-packages.yaml | 8 + .../externals/mpi/01-cce-ngtl-packages.yaml | 8 + .../externals/mpi/02-cce-ygtl-packages.yaml | 10 + .../rocm/00-version-543-packages.yaml | 73 +++ .../rocm/01-version-551-packages.yaml | 91 +++ var/sys_repo/systems/tioga/system.py | 130 +++++ 31 files changed, 2869 insertions(+), 535 deletions(-) create mode 100755 bin/benchpark-python create mode 100644 lib/benchpark/__init__.py create mode 100644 lib/benchpark/cmd/system.py create mode 100644 lib/benchpark/directives.py create mode 100644 lib/benchpark/error.py create mode 100644 lib/benchpark/experiment.py create mode 100644 lib/benchpark/paths.py create mode 100644 lib/benchpark/repo.py create mode 100644 lib/benchpark/runtime.py create mode 100644 lib/benchpark/spec.py create mode 100644 lib/benchpark/system.py create mode 100644 lib/benchpark/variant.py create mode 100755 lib/main.py create mode 100644 var/exp_repo/experiments/saxpy/experiment.py create mode 100644 var/exp_repo/repo.yaml create mode 100644 var/sys_repo/repo.yaml create mode 100644 var/sys_repo/systems/aws/system.py create mode 100644 var/sys_repo/systems/tioga/compilers/gcc/00-gcc-12-compilers.yaml create mode 100644 var/sys_repo/systems/tioga/compilers/rocm/00-rocm-551-compilers.yaml create mode 100644 var/sys_repo/systems/tioga/compilers/rocm/01-rocm-543-compilers.yaml create mode 100644 var/sys_repo/systems/tioga/externals/base/00-packages.yaml create mode 100644 var/sys_repo/systems/tioga/externals/libsci/00-gcc-packages.yaml create mode 100644 var/sys_repo/systems/tioga/externals/libsci/01-cce-packages.yaml create mode 100644 var/sys_repo/systems/tioga/externals/mpi/00-gcc-ngtl-packages.yaml create mode 100644 var/sys_repo/systems/tioga/externals/mpi/01-cce-ngtl-packages.yaml create mode 100644 var/sys_repo/systems/tioga/externals/mpi/02-cce-ygtl-packages.yaml create mode 100644 var/sys_repo/systems/tioga/externals/rocm/00-version-543-packages.yaml create mode 100644 var/sys_repo/systems/tioga/externals/rocm/01-version-551-packages.yaml create mode 100644 var/sys_repo/systems/tioga/system.py diff --git a/bin/benchpark b/bin/benchpark index 94813ac3a..dcbf5de63 100755 --- a/bin/benchpark +++ b/bin/benchpark @@ -5,546 +5,16 @@ # # SPDX-License-Identifier: Apache-2.0 -import argparse -from contextlib import contextmanager -import os +import os.path import pathlib -import shlex -import shutil import subprocess import sys -import yaml - -DEBUG = False - -__version__ = "0.1.0" - - -def debug_print(message): - if DEBUG: - print("(debug) " + str(message)) def main(): - if sys.version_info[:2] < (3, 8): - raise Exception("Benchpark requires at least python 3.8+.") - - parser = argparse.ArgumentParser(description="Benchpark") - parser.add_argument( - "-V", "--version", action="store_true", help="show version number and exit" - ) - - subparsers = parser.add_subparsers(title="Subcommands", dest="subcommand") - - actions = {} - benchpark_list(subparsers, actions) - benchpark_setup(subparsers, actions) - benchpark_tags(subparsers, actions) - - args = parser.parse_args() - no_args = True if len(sys.argv) == 1 else False - - if no_args: - parser.print_help() - return 1 - - if args.version: - print(get_version()) - return 0 - - if args.subcommand in actions: - actions[args.subcommand](args) - else: - print( - "Invalid subcommand ({args.subcommand}) - must choose one of: " - + " ".join(actions.keys()) - ) - - -def get_version(): - benchpark_version = __version__ - return benchpark_version - - -def source_location(): - script_location = os.path.dirname(os.path.abspath(__file__)) - return pathlib.Path(script_location).parent - - -def benchpark_list(subparsers, actions_dict): - list_parser = subparsers.add_parser( - "list", help="List available experiments, systems, and modifiers" - ) - list_parser.add_argument("sublist", nargs="?") - actions_dict["list"] = benchpark_list_handler - - -def benchpark_benchmarks(): - source_dir = source_location() - benchmarks = [] - experiments_dir = source_dir / "experiments" - for x in os.listdir(experiments_dir): - benchmarks.append(f"{x}") - return benchmarks - - -def benchpark_experiments(): - source_dir = source_location() - experiments = [] - experiments_dir = source_dir / "experiments" - for x in os.listdir(experiments_dir): - for y in os.listdir(experiments_dir / x): - experiments.append(f"{x}/{y}") - return experiments - - -def benchpark_systems(): - source_dir = source_location() - systems = [] - for x in os.listdir(source_dir / "configs"): - if not ( - os.path.isfile(os.path.join(source_dir / "configs", x)) or x == "common" - ): - systems.append(x) - return systems - - -def benchpark_modifiers(): - source_dir = source_location() - modifiers = [] - for x in os.listdir(source_dir / "modifiers"): - modifiers.append(x) - return modifiers - - -def benchpark_get_tags(): - f = source_location() / "tags.yaml" - tags = [] - - with open(f, "r") as stream: - try: - data = yaml.safe_load(stream) - except yaml.YAMLError as exc: - print(exc) - - for k0, v0 in data.items(): - if k0 == "benchpark-tags": - for k, v in v0.items(): - if isinstance(v, list): - for i in v: - tags.append(i) - else: - print("ERROR file does not contain benchpark-tags") - - return tags - - -def benchpark_list_handler(args): - source_dir = source_location() - sublist = args.sublist - benchmarks = benchpark_benchmarks() - experiments = benchpark_experiments() - systems = benchpark_systems() - modifiers = benchpark_modifiers() - - if sublist == None: - print("Experiments:") - for experiment in experiments: - print(f"\t{experiment}") - print("Systems:") - for system in systems: - print(f"\t{system}") - elif sublist == "benchmarks": - print("Benchmarks:") - for benchmark in benchmarks: - print(f"\t{benchmark}") - elif sublist == "experiments": - print("Experiments:") - for experiment in experiments: - print(f"\t{experiment}") - elif sublist == "systems": - print("Systems:") - for system in systems: - print(f"\t{system}") - elif sublist == "modifiers": - print("Modifiers:") - for modifier in modifiers: - print(f"\t{modifier}") - else: - raise ValueError( - f'Invalid benchpark list "{sublist}" - must choose [experiments], [systems], [modifiers] or leave empty' - ) - - -def benchpark_check_benchmark(arg_str): - benchmarks = benchpark_benchmarks() - found = arg_str in benchmarks - if not found: - out_str = f'Invalid benchmark "{arg_str}" - must choose one of: ' - for benchmark in benchmarks: - out_str += f"\n\t{benchmark}" - raise ValueError(out_str) - return found - - -def benchpark_check_experiment(arg_str): - experiments = benchpark_experiments() - found = arg_str in experiments - if not found: - out_str = f'Invalid experiment (benchmark/ProgrammingModel) "{arg_str}" - must choose one of: ' - for experiment in experiments: - out_str += f"\n\t{experiment}" - raise ValueError(out_str) - return found - - -def benchpark_check_system(arg_str): - systems = benchpark_systems() - found = arg_str in systems - if not found: - out_str = f'Invalid system "{arg_str}" - must choose one of: ' - for system in systems: - out_str += f"\n\t{system}" - raise ValueError(out_str) - return found - - -def benchpark_check_tag(arg_str): - tags = benchpark_get_tags() - found = arg_str in tags - if not found: - out_str = f'Invalid tag "{arg_str}" - must choose one of: ' - for tag in tags: - out_str += f"\n\t{tag}" - raise ValueError(out_str) - return found - - -def benchpark_check_modifier(arg_str): - modifiers = benchpark_modifiers() - found = arg_str in modifiers - if not found: - out_str = f'Invalid modifier "{arg_str}" - must choose one of: ' - for modifier in modifiers: - out_str += f"\n\t{modifier}" - raise ValueError(out_str) - return found - - -def benchpark_setup(subparsers, actions_dict): - create_parser = subparsers.add_parser( - "setup", help="Set up an experiment and prepare it to build/run" - ) - - create_parser.add_argument( - "experiment", - type=str, - help="The experiment (benchmark/ProgrammingModel) to run", - ) - create_parser.add_argument( - "system", type=str, help="The system on which to run the experiment" - ) - create_parser.add_argument( - "experiments_root", - type=str, - help="Where to install packages and store results for the experiments. Benchpark expects to manage this directory, and it should be empty/nonexistent the first time you run benchpark setup experiments.", - ) - create_parser.add_argument( - "--modifier", - type=str, - default="none", - help="The modifier to apply to the experiment (default none)", - ) - - actions_dict["setup"] = benchpark_setup_handler - - -def run_command(command_str, env=None): - proc = subprocess.Popen( - shlex.split(command_str), - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - stdout, stderr = proc.communicate() - if proc.returncode != 0: - raise RuntimeError( - f"Failed command: {command_str}\nOutput: {stdout}\nError: {stderr}" - ) - - return (stdout, stderr) - - -def benchpark_tags(subparsers, actions_dict): - create_parser = subparsers.add_parser("tags", help="Tags in Benchpark experiments") - create_parser.add_argument( - "experiments_root", - type=str, - help="The experiments_root you specified during Benchpark setup.", - ) - create_parser.add_argument( - "-a", - "--application", - action="store", - help="The application for which to find Benchpark tags", - ) - create_parser.add_argument( - "-t", - "--tag", - action="store", - help="The tag for which to search in Benchpark experiments", - ) - actions_dict["tags"] = benchpark_tags_handler - - -# Note: it would be nice to vendor spack.llnl.util.link_tree, but that -# involves pulling in most of llnl/util/ and spack/util/ -def symlink_tree(src, dst, include_fn=None): - """Like ``cp -R`` but instead of files, create symlinks""" - src = os.path.abspath(src) - dst = os.path.abspath(dst) - # By default, we include all filenames - include_fn = include_fn or (lambda f: True) - for x in [src, dst]: - if not os.path.isdir(x): - raise ValueError(f"Not a directory: {x}") - for src_subdir, directories, files in os.walk(src): - relative_src_dir = pathlib.Path(os.path.relpath(src_subdir, src)) - dst_dir = pathlib.Path(dst) / relative_src_dir - dst_dir.mkdir(parents=True, exist_ok=True) - for x in files: - if not include_fn(x): - continue - dst_symlink = dst_dir / x - src_file = os.path.join(src_subdir, x) - os.symlink(src_file, dst_symlink) - - -@contextmanager -def working_dir(location): - initial_dir = os.getcwd() - try: - os.chdir(location) - yield - finally: - os.chdir(initial_dir) - - -def git_clone_commit(url, commit, destination): - run_command(f"git clone -c feature.manyFiles=true {url} {destination}") - - with working_dir(destination): - run_command(f"git checkout {commit}") - - -def benchpark_setup_handler(args): - """ - experiments_root/ - spack/ - ramble/ - / - / - workspace/ - configs/ - (everything from source/configs/) - (everything from source/experiments/) - """ - - experiment = args.experiment - system = args.system - experiments_root = pathlib.Path(os.path.abspath(args.experiments_root)) - modifier = args.modifier - source_dir = source_location() - debug_print(f"source_dir = {source_dir}") - debug_print(f"specified experiment (benchmark/ProgrammingModel) = {experiment}") - valid_experiment = benchpark_check_experiment(experiment) - debug_print(f"specified system = {system}") - valid_system = benchpark_check_system(system) - debug_print(f"specified modifier = {modifier}") - valid_modifier = benchpark_check_modifier(modifier) - - workspace_dir = experiments_root / str(experiment) / str(system) - - if workspace_dir.exists(): - if workspace_dir.is_dir(): - print(f"Clearing existing workspace {workspace_dir}") - shutil.rmtree(workspace_dir) - else: - print( - f"Benchpark expects to manage {workspace_dir} as a directory, but it is not" - ) - sys.exit(1) - - workspace_dir.mkdir(parents=True) - - ramble_workspace_dir = workspace_dir / "workspace" - ramble_configs_dir = ramble_workspace_dir / "configs" - ramble_logs_dir = ramble_workspace_dir / "logs" - ramble_spack_experiment_configs_dir = ( - ramble_configs_dir / "auxiliary_software_files" - ) - - print(f"Setting up configs for Ramble workspace {ramble_configs_dir}") - - configs_src_dir = source_dir / "configs" / str(system) - experiment_src_dir = source_dir / "experiments" / experiment - modifier_config_dir = source_dir / "modifiers" / modifier / "configs" - ramble_configs_dir.mkdir(parents=True) - ramble_logs_dir.mkdir(parents=True) - ramble_spack_experiment_configs_dir.mkdir(parents=True) - - def include_fn(fname): - # Only include .yaml and .tpl files - # Always exclude files that start with "." - if fname.startswith("."): - return False - if fname.endswith(".yaml"): - return True - return False - - symlink_tree(configs_src_dir, ramble_configs_dir, include_fn) - symlink_tree(experiment_src_dir, ramble_configs_dir, include_fn) - symlink_tree(modifier_config_dir, ramble_configs_dir, include_fn) - symlink_tree( - source_dir / "configs" / "common", - ramble_spack_experiment_configs_dir, - include_fn, - ) - - template_name = "execute_experiment.tpl" - experiment_template_options = [ - configs_src_dir / template_name, - experiment_src_dir / template_name, - source_dir / "common-resources" / template_name, - ] - for choice_template in experiment_template_options: - if os.path.exists(choice_template): - break - os.symlink( - choice_template, - ramble_configs_dir / "execute_experiment.tpl", - ) - - spack_location = experiments_root / "spack" - ramble_location = experiments_root / "ramble" - - spack_exe = spack_location / "bin" / "spack" - ramble_exe = ramble_location / "bin" / "ramble" - spack_cache_location = spack_location / "misc-cache" - - initializer_script = experiments_root / "setup.sh" - - checkout_versions_location = source_dir / "checkout-versions.yaml" - with open(checkout_versions_location, "r") as yaml_file: - data = yaml.safe_load(yaml_file) - ramble_commit = data["versions"]["ramble"] - spack_commit = data["versions"]["spack"] - - if not spack_location.exists(): - print(f"Cloning spack into {spack_location}") - git_clone_commit( - "https://github.com/spack/spack.git", spack_commit, spack_location - ) - - env = {"SPACK_DISABLE_LOCAL_CONFIG": "1"} - run_command( - f"{spack_exe} config --scope=site add config:misc_cache:{spack_cache_location}", - env=env, - ) - run_command(f"{spack_exe} repo add --scope=site {source_dir}/repo", env=env) - - if not ramble_location.exists(): - print(f"Cloning ramble into {ramble_location}") - git_clone_commit( - "https://github.com/GoogleCloudPlatform/ramble.git", - ramble_commit, - ramble_location, - ) - - run_command(f"{ramble_exe} repo add --scope=site {source_dir}/repo") - run_command( - f'{ramble_exe} config --scope=site add "config:disable_progress_bar:true"' - ) - run_command( - f"{ramble_exe} repo add -t modifiers --scope=site {source_dir}/modifiers" - ) - run_command( - f"{ramble_exe} config --scope=site add \"config:spack:global:args:'-d'\"" - ) - - if not initializer_script.exists(): - with open(initializer_script, "w") as f: - f.write( - f"""\ -if [ -n "${{_BENCHPARK_INITIALIZED:-}}" ]; then - return 0 -fi - -. {spack_location}/share/spack/setup-env.sh -. {ramble_location}/share/ramble/setup-env.sh - -export SPACK_DISABLE_LOCAL_CONFIG=1 - -export _BENCHPARK_INITIALIZED=true -""" - ) - - instructions = f"""\ -To complete the benchpark setup, do the following: - - . {initializer_script} - -Further steps are needed to build the experiments (ramble -P -D {ramble_workspace_dir} workspace setup) and run them (ramble -P -D {ramble_workspace_dir} on) -""" - print(instructions) - - -def helper_experiments_tags(ramble_exe, benchmarks): - # find all tags in Ramble applications (both in Ramble built-in and in Benchpark/repo) - (tags_stdout, tags_stderr) = run_command(f"{ramble_exe} attributes --tags --all") - ramble_applications_tags = {} - lines = tags_stdout.splitlines() - - for line in lines: - key_value = line.split(":") - ramble_applications_tags[key_value[0]] = key_value[1].strip().split(",") - - benchpark_experiments_tags = {} - for benchmark in benchmarks: - benchpark_experiments_tags[benchmark] = ramble_applications_tags[benchmark] - return benchpark_experiments_tags - - -def benchpark_tags_handler(args): - """ - Filter ramble tags by benchpark benchmarks - """ - source_dir = source_location() - experiments_root = pathlib.Path(os.path.abspath(args.experiments_root)) - ramble_location = experiments_root / "ramble" - ramble_exe = ramble_location / "bin" / "ramble" - benchmarks = benchpark_benchmarks() - - if args.tag: - if benchpark_check_tag(args.tag): - # find all applications in Ramble that have a given tag (both in Ramble built-in and in Benchpark/repo) - (tag_stdout, tag_stderr) = run_command(f"{ramble_exe} list -t {args.tag}") - lines = tag_stdout.splitlines() - - for line in lines: - if line in benchmarks: - print(line) - - elif args.application: - if benchpark_check_benchmark(args.application): - benchpark_experiments_tags = helper_experiments_tags(ramble_exe, benchmarks) - print(benchpark_experiments_tags[args.application]) - else: - benchpark_experiments_tags = helper_experiments_tags(ramble_exe, benchmarks) - print("All tags that exist in Benchpark experiments:") - for k, v in benchpark_experiments_tags.items(): - print(k) + basedir = pathlib.Path(os.path.abspath(__file__)).parent.parent + main = basedir / "lib" / "main.py" + subprocess.run([sys.executable, main] + sys.argv[1:], check=True) if __name__ == "__main__": diff --git a/bin/benchpark-python b/bin/benchpark-python new file mode 100755 index 000000000..a7fd9c642 --- /dev/null +++ b/bin/benchpark-python @@ -0,0 +1,21 @@ +#!/bin/sh +# +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +# +# benchpark-python +# +# If you want to write your own executable Python script that uses Benchpark +# modules, on Mac OS or maybe some others, you may be able to do it like +# this: +# +# #!/usr/bin/env benchpark-python +# +# This is compatible across platforms. +# +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +export PYTHONPATH="${SCRIPT_DIR}/../lib":$PYTHONPATH +exec python3 -i "$@" diff --git a/lib/benchpark/__init__.py b/lib/benchpark/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/benchpark/cmd/system.py b/lib/benchpark/cmd/system.py new file mode 100644 index 000000000..57fb2d802 --- /dev/null +++ b/lib/benchpark/cmd/system.py @@ -0,0 +1,68 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +import shutil +import sys + +import benchpark.system +import benchpark.spec + + +def system_init(args): + system_spec = benchpark.spec.SystemSpec(" ".join(args.spec)) + system_spec = system_spec.concretize() + + system = system_spec.system + system.initialize() + + if args.basedir: + base = args.basedir + sysdir = system.system_id() + destdir = os.path.join(base, sysdir) + elif args.dest: + destdir = args.dest + else: + raise ValueError("Must specify one of: --dest, --basedir") + + try: + os.mkdir(destdir) + system.generate_description(destdir) + except FileExistsError: + print(f"Abort: system description dir already exists ({destdir})") + sys.exit(1) + except Exception: + # If there was a failure, remove any partially-generated resources + shutil.rmtree(destdir) + raise + + +def system_list(args): + raise NotImplementedError("'benchpark system list' is not available") + + +def setup_parser(root_parser): + system_subparser = root_parser.add_subparsers(dest="system_subcommand") + + init_parser = system_subparser.add_parser("init") + init_parser.add_argument("--dest", help="Place all system files here directly") + init_parser.add_argument( + "--basedir", help="Generate a system dir under this, and place all files there" + ) + + init_parser.add_argument("spec", nargs="+", help="System spec") + + system_subparser.add_parser("list") + + +def command(args): + actions = { + "init": system_init, + "list": system_list, + } + if args.system_subcommand in actions: + actions[args.system_subcommand](args) + else: + raise ValueError(f"Unknown subcommand for 'system': {args.system_subcommand}") diff --git a/lib/benchpark/directives.py b/lib/benchpark/directives.py new file mode 100644 index 000000000..d930a7652 --- /dev/null +++ b/lib/benchpark/directives.py @@ -0,0 +1,202 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +import collections.abc +import inspect +import os +import re +from typing import Any, Callable, Optional, Tuple, Union + +import benchpark.spec +import benchpark.paths +import benchpark.repo +import benchpark.runtime +import benchpark.variant + +import ramble.language.language_base +import ramble.language.language_helpers +import ramble.language.shared_language +from ramble.language.language_base import DirectiveError + + +# TODO remove this when it is added to ramble.lang (when ramble updates from spack) +class classproperty: + """Non-data descriptor to evaluate a class-level property. The function that performs + the evaluation is injected at creation time and take an instance (could be None) and + an owner (i.e. the class that originated the instance) + """ + + def __init__(self, callback): + self.callback = callback + + def __get__(self, instance, owner): + return self.callback(owner) + + +class DirectiveMeta(ramble.language.shared_language.SharedMeta): + """ + metaclass for supporting directives (e.g., depends_on) and phases + """ + + _directive_names = set() + _directives_to_be_executed = [] + + # Hack to be able to use SharedMeta outside of Ramble + # will ask Ramble to implement fix on their end and then we can remove this + def __init__(self, *args, **kwargs): + with benchpark.repo.override_ramble_hardcoded_globals(): + super(DirectiveMeta, self).__init__(*args, **kwargs) + + +benchpark_directive = DirectiveMeta.directive + + +@benchpark_directive("variants") +def variant( + name: str, + default: Optional[Any] = None, + description: str = "", + values: Optional[Union[collections.abc.Sequence, Callable[[Any], bool]]] = None, + multi: Optional[bool] = None, + validator: Optional[Callable[[str, str, Tuple[Any, ...]], None]] = None, + when: Optional[Union[str, bool]] = None, + sticky: bool = False, +): + """Define a variant. + Can specify a default value as well as a text description. + Args: + name: Name of the variant + default: Default value for the variant, if not specified otherwise the default will be + False for a boolean variant and 'nothing' for a multi-valued variant + description: Description of the purpose of the variant + values: Either a tuple of strings containing the allowed values, or a callable accepting + one value and returning True if it is valid + multi: If False only one value per spec is allowed for this variant + validator: Optional group validator to enforce additional logic. It receives the experiment + name, the variant name and a tuple of values and should raise an instance of BenchparkError + if the group doesn't meet the additional constraints + when: Optional condition on which the variant applies + sticky: The variant should not be changed by the concretizer to find a valid concrete spec + Raises: + DirectiveError: If arguments passed to the directive are invalid + """ + + def format_error(msg, pkg): + msg += " @*r{{[{0}, variant '{1}']}}" + return msg.format(pkg.name, name) + + def _always_true(_x): + return True + + # Ensure we have a sequence of allowed variant values, or a + # predicate for it. + if values is None: + if str(default).upper() in ("TRUE", "FALSE"): + values = (True, False) + else: + values = _always_true + + # The object defining variant values might supply its own defaults for + # all the other arguments. Ensure we have no conflicting definitions + # in place. + for argument in ("default", "multi", "validator"): + # TODO: we can consider treating 'default' differently from other + # TODO: attributes and let a packager decide whether to use the fluent + # TODO: interface or the directive argument + if hasattr(values, argument) and locals()[argument] is not None: + + def _raise_argument_error(pkg): + msg = ( + "Remove specification of {0} argument: it is handled " + "by an attribute of the 'values' argument" + ) + raise DirectiveError(format_error(msg.format(argument), pkg)) + + return _raise_argument_error + + # Allow for the object defining the allowed values to supply its own + # default value and group validator, say if it supports multiple values. + default = getattr(values, "default", default) + validator = getattr(values, "validator", validator) + multi = getattr(values, "multi", bool(multi)) + + # Here we sanitize against a default value being either None + # or the empty string, as the former indicates that a default + # was not set while the latter will make the variant unparsable + # from the command line + if default is None or default == "": + + def _raise_default_not_set(pkg): + if default is None: + msg = "either a default was not explicitly set, or 'None' was used" + elif default == "": + msg = "the default cannot be an empty string" + raise DirectiveError(format_error(msg, pkg)) + + return _raise_default_not_set + + description = str(description).strip() + + def _execute_variant(pkg): + if not re.match(benchpark.spec.IDENTIFIER, name): + directive = "variant" + msg = "Invalid variant name in {0}: '{1}'" + raise DirectiveError(directive, msg.format(pkg.name, name)) + + pkg.variants[name] = benchpark.variant.Variant( + name, default, description, values, multi, validator, sticky + ) + + return _execute_variant + + +class ExperimentSystemBase(metaclass=DirectiveMeta): + @classproperty + def template_dir(cls): + """Directory where the experiment/system.py file lives.""" + return os.path.abspath(os.path.dirname(cls.module.__file__)) + + @classproperty + def module(cls): + """Module object (not just the name) that this Experiment/System is + defined in. + """ + return __import__(cls.__module__, fromlist=[cls.__name__]) + + @classproperty + def namespace(cls): + """namespace for the Experiment/System, which identifies its repo.""" + parts = cls.__module__.split(".") + return ".".join(parts[2:-1]) + + @classproperty + def fullname(cls): + """Name of this Experiment/System, including the namespace""" + return f"{cls.namespace}.{cls.name}" + + @classproperty + def fullnames(cls): + """Fullnames for this Experiment/System and any from which it inherits.""" + fullnames = [] + for cls in inspect.getmro(cls): + namespace = getattr(cls, "namespace", None) + if namespace: + fullnames.append(f"{namespace}.{cls.name}") + if namespace == "builtin": + # builtin packages cannot inherit from other repos + break + return fullnames + + @classproperty + def name(cls): + """The name of this Experiment/System. + This is the name of its Python module, without the containing module + names. + """ + if cls._name is None: + cls._name = cls.module.__name__ + if "." in cls._name: + cls._name = cls._name[cls._name.rindex(".") + 1 :] + return cls._name diff --git a/lib/benchpark/error.py b/lib/benchpark/error.py new file mode 100644 index 000000000..bb604536e --- /dev/null +++ b/lib/benchpark/error.py @@ -0,0 +1,14 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +#: at what level we should write stack traces or short error messages +#: this is module-scoped because it needs to be set very early +debug = 0 + + +class BenchparkError(Exception): + """This is the superclass for all Benchpark errors. + Subclasses can be found in the modules they have to do with. + """ diff --git a/lib/benchpark/experiment.py b/lib/benchpark/experiment.py new file mode 100644 index 000000000..4ee064d3c --- /dev/null +++ b/lib/benchpark/experiment.py @@ -0,0 +1,98 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Dict, Tuple +import yaml # TODO: some way to ensure yaml available + +from benchpark.directives import ExperimentSystemBase +import benchpark.spec +import benchpark.paths +import benchpark.repo +import benchpark.runtime +import benchpark.variant + +bootstrapper = benchpark.runtime.RuntimeResources(benchpark.paths.benchpark_home) +bootstrapper.bootstrap() + +import ramble.language.language_base # noqa +import ramble.language.language_helpers # noqa + + +class Experiment(ExperimentSystemBase): + """This is the superclass for all benchpark experiments. + + ***The Experiment class*** + + Experiments are written in pure Python. + + There are two main parts of a Benchpark experiment: + + 1. **The experiment class**. Classes contain ``directives``, which are + special functions, that add metadata (variants) to packages (see + ``directives.py``). + + 2. **Experiment instances**. Once instantiated, an experiment is + essentially a collection of files defining an experiment in a + Ramble workspace. + """ + + # + # These are default values for instance variables. + # + + # This allows analysis tools to correctly interpret the class attributes. + variants: Dict[ + str, + Tuple["benchpark.variant.Variant", "benchpark.spec.ConcreteExperimentSpec"], + ] + + def __init__(self, spec): + self.spec: "benchpark.spec.ConcreteExperimentSpec" = spec + super().__init__() + + def compute_include_section(self): + # include the config directory + # TODO: does this need to change to interop with System class + return ["./configs"] + + def compute_config_section(self): + # default configs for all experiments + return { + "deprecated": True, + "spack_flags": {"install": "--add --keep-stage", "concretize": "-U -f"}, + } + + def compute_modifiers_section(self): + # by default we use the allocation modifier and no others + return [{"name": "allocation"}] + + def compute_applications_section(self): + # TODO: is there some reasonable default? + raise NotImplementedError( + "Each experiment must implement compute_applications_section" + ) + + def compute_spack_section(self): + # TODO: is there some reasonable default based on known variable names? + raise NotImplementedError( + "Each experiment must implement compute_spack_section" + ) + + def compute_ramble_dict(self): + # This can be overridden by any subclass that needs more flexibility + return { + "ramble": { + "include": self.compute_include_section(), + "config": self.compute_config_section(), + "modifiers": self.compute_modifiers_section(), + "applications": self.compute_applications_section(), + "spack": self.compute_spack_section(), + } + } + + def write_ramble_dict(self, filepath): + ramble_dict = self.compute_ramble_dict() + with open(filepath, "w") as f: + yaml.dump(ramble_dict, f) diff --git a/lib/benchpark/paths.py b/lib/benchpark/paths.py new file mode 100644 index 000000000..16ec3cf7f --- /dev/null +++ b/lib/benchpark/paths.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 + +import os +import pathlib + +benchpark_home = pathlib.Path(os.path.expanduser("~/.benchpark")) +global_ramble_path = benchpark_home / "ramble" +global_spack_path = benchpark_home / "spack" diff --git a/lib/benchpark/repo.py b/lib/benchpark/repo.py new file mode 100644 index 000000000..a413b47bf --- /dev/null +++ b/lib/benchpark/repo.py @@ -0,0 +1,201 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import sys +import contextlib +import pathlib + +from enum import Enum + +import benchpark.paths +import benchpark.runtime + +# isort: off + +bootstrapper = benchpark.runtime.RuntimeResources( + benchpark.paths.benchpark_home +) # noqa +bootstrapper.bootstrap() # noqa + +import llnl.util.lang # noqa +import ramble.language.language_base # noqa +import ramble.repository # noqa + +# isort: on + +global_namespace = "benchpark" +namespaces = ["benchpark.expr", "benchpark.sys"] + +#: Guaranteed unused default value for some functions. +NOT_PROVIDED = object() + + +#### +# Implement type specific functionality between here, and +# END TYPE SPECIFIC FUNCTIONALITY +#### +ObjectTypes = Enum("ObjectTypes", ["experiments", "systems"]) + +OBJECT_NAMES = [obj.name for obj in ObjectTypes] + +default_type = ObjectTypes.experiments + +type_definitions = { + ObjectTypes.experiments: { + "file_name": "experiment.py", + "dir_name": "experiments", + "abbrev": "expr", + "config_section": "repos", + "accepted_configs": ["repo.yaml"], + "singular": "experiment", + }, + ObjectTypes.systems: { + "file_name": "system.py", + "dir_name": "systems", + "abbrev": "sys", + "config_section": "repos", + "accepted_configs": ["repo.yaml"], + "singular": "system", + }, +} + + +@contextlib.contextmanager +def override_ramble_hardcoded_globals(): + _old = ( + ramble.repository.type_definitions, + ramble.repository.global_namespace, + ramble.language.language_base.namespaces, + ) + ramble.repository.type_definitions = type_definitions + ramble.repository.global_namespace = global_namespace + ramble.language.language_base.namespaces = namespaces + + yield + + ramble.repository.type_definitions = _old[0] + ramble.repository.global_namespace = _old[1] + ramble.language.language_base.namespaces = _old[2] + + +def _base_path(): + return pathlib.Path(__file__).resolve().parents[2] + + +# Experiments +def _exprs(): + """Get the singleton RepoPath instance for Ramble. + + Create a RepoPath, add it to sys.meta_path, and return it. + + TODO: consider not making this a singleton. + """ + experiments_repo = _base_path() / "var" / "exp_repo" + return _add_repo(experiments_repo, ObjectTypes.experiments) + + +def _add_repo(repo_dir, obj_type): + if repo_dir.exists(): + repo_dirs = [str(repo_dir)] + else: + raise ValueError(f"Repo dir does not exist: {repo_dir}") + + with override_ramble_hardcoded_globals(): + path = ramble.repository.RepoPath(*repo_dirs, object_type=obj_type) + sys.meta_path.append(path) + return path + + +# Systems +def _systems(): + systems_repo = _base_path() / "var" / "sys_repo" + return _add_repo(systems_repo, ObjectTypes.systems) + + +paths = { + ObjectTypes.experiments: llnl.util.lang.Singleton(_exprs), + ObjectTypes.systems: llnl.util.lang.Singleton(_systems), +} + + +##################################### +# END TYPE SPECIFIC FUNCTIONALITY +##################################### + + +def all_object_names(object_type=default_type): + """Convenience wrapper around ``ramble.repository.all_object_names()``.""" # noqa: E501 + return paths[object_type].all_object_names() + + +def get(spec, object_type=default_type): + """Convenience wrapper around ``ramble.repository.get()``.""" + return paths[object_type].get(spec) + + +def set_path(repo, object_type=default_type): + """Set the path singleton to a specific value. + + Overwrite ``path`` and register it as an importer in + ``sys.meta_path`` if it is a ``Repo`` or ``RepoPath``. + """ + global paths + paths[object_type] = repo + + # make the new repo_path an importer if needed + append = isinstance(repo, (ramble.repository.Repo, ramble.repository.RepoPath)) + if append: + sys.meta_path.append(repo) + return append + + +@contextlib.contextmanager +def additional_repository(repository, object_type=default_type): + """Adds temporarily a repository to the default one. + + Args: + repository: repository to be added + """ + paths[object_type].put_first(repository) + yield + paths[object_type].remove(repository) + + +@contextlib.contextmanager +def use_repositories(*paths_and_repos, object_type=default_type): + """Use the repositories passed as arguments within the context manager. + + Args: + *paths_and_repos: paths to the repositories to be used, or + already constructed Repo objects + + Returns: + Corresponding RepoPath object + """ + global paths + + # Construct a temporary RepoPath object from + temporary_repositories = ramble.repository.RepoPath( + *paths_and_repos, object_type=object_type + ) + + # Swap the current repository out + saved = paths[object_type] + remove_from_meta = set_path(temporary_repositories, object_type=object_type) + + yield temporary_repositories + + # Restore _path and sys.meta_path + if remove_from_meta: + sys.meta_path.remove(temporary_repositories) + paths[object_type] = saved + + +# Add the finder to sys.meta_path +REPOS_FINDER = ramble.repository.ReposFinder() +sys.meta_path.append(REPOS_FINDER) diff --git a/lib/benchpark/runtime.py b/lib/benchpark/runtime.py new file mode 100644 index 000000000..74638dd77 --- /dev/null +++ b/lib/benchpark/runtime.py @@ -0,0 +1,147 @@ +from contextlib import contextmanager +import os +import pathlib +import shlex +import subprocess +import sys +import yaml + +DEBUG = False + + +def debug_print(message): + if DEBUG: + print("(debug) " + str(message)) + + +@contextmanager +def working_dir(location): + initial_dir = os.getcwd() + try: + os.chdir(location) + yield + finally: + os.chdir(initial_dir) + + +def git_clone_commit(url, commit, destination): + run_command(f"git clone -c feature.manyFiles=true {url} {destination}") + + with working_dir(destination): + run_command(f"git checkout {commit}") + + +def benchpark_root(): + this_module_path = pathlib.Path(os.path.abspath(__file__)) + return this_module_path.parents[2] + + +def run_command(command_str, env=None): + proc = subprocess.Popen( + shlex.split(command_str), + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + stdout, stderr = proc.communicate() + if proc.returncode != 0: + raise RuntimeError( + f"Failed command: {command_str}\nOutput: {stdout}\nError: {stderr}" + ) + + return (stdout, stderr) + + +class Command: + def __init__(self, exe_path, env): + self.exe_path = exe_path + self.env = env + + def __call__(self, *args): + opts_str = " ".join(args) + cmd_str = f"{self.exe_path} {opts_str}" + return run_command(cmd_str, env=self.env) + + +class RuntimeResources: + def __init__(self, dest): + self.root = benchpark_root() + self.dest = pathlib.Path(dest) + + checkout_versions_location = self.root / "checkout-versions.yaml" + with open(checkout_versions_location, "r") as yaml_file: + data = yaml.safe_load(yaml_file) + self.ramble_commit = data["versions"]["ramble"] + self.spack_commit = data["versions"]["spack"] + + self.ramble_location = self.dest / "ramble" + self.spack_location = self.dest / "spack" + + def bootstrap(self): + if not self.ramble_location.exists(): + self._install_ramble() + ramble_lib_path = self.ramble_location / "lib" / "ramble" + externals = str(ramble_lib_path / "external") + if externals not in sys.path: + sys.path.insert(1, externals) + internals = str(ramble_lib_path) + if internals not in sys.path: + sys.path.insert(1, internals) + + # Spack does not go in sys.path, but we will manually access modules from it + # The reason for this oddity is that spack modules will compete with the internal + # spack modules from ramble + if not self.spack_location.exists(): + self._install_spack() + + def _install_ramble(self): + debug_print(f"Cloning Ramble to {self.ramble_location}") + git_clone_commit( + "https://github.com/GoogleCloudPlatform/ramble.git", + self.ramble_commit, + self.ramble_location, + ) + debug_print(f"Done cloning Ramble ({self.ramble_location})") + + def _install_spack(self): + debug_print(f"Cloning Spack to {self.spack_location}") + git_clone_commit( + "https://github.com/spack/spack.git", self.spack_commit, self.spack_location + ) + debug_print(f"Done cloning Spack ({self.spack_location})") + + def _ramble(self): + first_time = False + if not self.ramble_location.exists(): + first_time = True + self._install_ramble() + return Command(self.ramble_location / "bin" / "ramble", env={}), first_time + + def _spack(self): + env = {"SPACK_DISABLE_LOCAL_CONFIG": "1"} + spack = Command(self.spack_location / "bin" / "spack", env) + spack_cache_location = self.spack_location / "misc-cache" + first_time = False + if not self.spack_location.exists(): + first_time = True + self._install_spack() + spack( + "config", + "--scope=site", + "add", + f"config:misc_cache:{spack_cache_location}", + ) + return spack, first_time + + def spack_first_time_setup(self): + return self._spack() + + def ramble_first_time_setup(self): + return self._ramble() + + def spack(self): + return self._spack()[0] + + def ramble(self): + return self._ramble()[0] diff --git a/lib/benchpark/spec.py b/lib/benchpark/spec.py new file mode 100644 index 000000000..eaa56433d --- /dev/null +++ b/lib/benchpark/spec.py @@ -0,0 +1,548 @@ +import enum +import functools +import pathlib +import re +from typing import Iterable, Iterator, List, Match, Optional, Union + +import benchpark.paths +import benchpark.repo +import benchpark.runtime + +bootstrapper = benchpark.runtime.RuntimeResources(benchpark.paths.benchpark_home) +bootstrapper.bootstrap() + +import llnl.util.lang # noqa + +repo_path = benchpark.repo.paths[benchpark.repo.ObjectTypes.experiments] +sys_repo = benchpark.repo.paths[benchpark.repo.ObjectTypes.systems] + + +class VariantMap(llnl.util.lang.HashableMap): + def __init__(self, init: "VariantMap" = None): + super().__init__() + if init: + self.dict = init.dict.copy() + + def __setitem__(self, name: str, values: Union[str, Iterable]): + if name in self.dict: + raise Exception(f"Cannot specify variant {name} twice") + if isinstance(values, str): + values = (values,) + else: + values = tuple(*values) + super().__setitem__(name, values) + + def intersects(self, other: "VariantMap") -> bool: + if isinstance(other, ConcreteVariantMap): + return other.intersects(self) + + # always possible to constrain since abstract variants are multi-value + return True + + def satisfies(self, other: "VariantMap") -> bool: + if isinstance(other, ConcreteVariantMap): + self == other + + return all( + name in self and set(self[name]) >= set(other[name]) for name in other + ) + + @staticmethod + def stringify(name: str, values: tuple) -> str: + if len(values) == 1: + if values[0].lower() == "true": + return f"+{name}" + if values[0].lower() == "false": + return f"~{name}" + return f"{name}={','.join(values)}" + + def __str__(self): + return " ".join( + self.stringify(name, values) for name, values in self.dict.items() + ) + + +class ConcreteVariantMap(VariantMap): + def __setitem__(self, name, values): + raise TypeError(f"{self.__class__} is immutable.") + + def intersects(self, other: VariantMap) -> bool: + return self.satisfies(other) + + +class Spec(object): + def __init__(self, str_or_spec: Optional[Union[str, "Spec"]] = None): + self._name = None + self._namespace = None + self._variants = VariantMap() + + if isinstance(str_or_spec, Spec): + self._dup(str_or_spec) + elif isinstance(str_or_spec, str): + self._parse(str_or_spec) + elif str_or_spec is not None: + msg = f"{self.__class__} can only be instantiated from {self.__class__} or str, " + msg += f"not from {type(str_or_spec)}." + raise NotImplementedError(msg) + + # getter/setter for each attribute so that ConcreteSpec can be immutable + @property + def name(self): + return self._name + + @name.setter + def name(self, value: str): + self._name = value + + @property + def namespace(self): + return self._namespace + + @namespace.setter + def namespace(self, value: str): + self._namespace = value + + @property + def variants(self): + return self._variants + + # This one is probably unnecessary, but here for completeness + @variants.setter + def variants(self, value: VariantMap): + self._variants = value + + def __eq__(self, other: "Spec"): + if other is None: + return False + + return ( + self.name == other.name + and ( + self.namespace is None + or other.namespace is None + or self.namespace == other.namespace + ) + and self.variants == other.variants + ) + + def __str__(self): + string = "" + if self.namespace is not None: + string += f"{self.namespace}." + if self.name is not None: + string += self.name + + variants = str(self.variants) + if not string: + return variants + string += f" {variants}" if variants else "" + return string + + def __repr__(self): + return str(self) + + def _dup(self, other: "Spec"): + # operate on underlying types so it can be called on ConcreteSpec + self._name = other.name + self._namespace = other.namespace + self._variants = other.variants + + def _parse(self, string: str): + specs = SpecParser( + type(self), string + ).all_specs() # parse spec of appropriate type + assert len(specs) == 1, f"{string} does not parse to one spec" + + self._dup(specs[0]) + + def intersects(self, other: Union[str, "Spec"]) -> bool: + if not isinstance(other, Spec): + other = type(self)(other) # keep type from subclass + return ( + (self.name is None or other.name is None or self.name == other.name) + and ( + self.namespace is None + or other.namespace is None + or self.namespace == other.namespace + ) + and self.variants.intersects(other.variants) + ) + + def satisfies(self, other: Union[str, "Spec"]) -> bool: + if not isinstance(other, Spec): + other = type(self)(other) # keep type from subclass + return ( + (other.name is None or self.name == other.name) + and (other.namespace is None or self.namespace == other.namespace) + and self.variants.satisfies(other.variants) + ) + + def concretize(self): + raise NotImplementedError("Spec.concretize must be implemented by subclass") + + @property + def object_class(self): + raise NotImplementedError( + f"{type(self)} does not implement object_class property" + ) + + +class ExperimentSpec(Spec): + @property + def experiment_class(self): + return repo_path.get_obj_class(self.name) + + @property + def object_class(self): + # shared getter so that multiple spec types can be concretized similarly + return self.experiment_class + + def concretize(self): + return ConcreteExperimentSpec(self) + + +def autospec(function): + """Decorator that automatically converts the first argument of a + function to a Spec. + """ + + @functools.wraps(function) + def converter(self, spec_like, *args, **kwargs): + if not isinstance(spec_like, ExperimentSpec): + spec_like = ExperimentSpec(spec_like) + return function(self, spec_like, *args, **kwargs) + + return converter + + +class ConcreteSpec(Spec): + def __init__(self, str_or_spec: Union[str, Spec]): + super().__init__(str_or_spec) + self._concretize() + + def __hash__(self): + return hash((self.name, self.namespace, self.variants)) + + @property + def name(self): + return self._name + + @name.setter + def name(self, value: str): + raise TypeError(f"{self.__class__} is immutable") + + @property + def namespace(self): + return self._namespace + + @namespace.setter + def namespace(self, value: str): + raise TypeError(f"{self.__class__} is immutable") + + @property + def variants(self): + return self._variants + + @variants.setter + def variants(self, value: str): + raise TypeError(f"{self.__class__} is immutable") + + def _concretize(self): + if not self.name: + raise AnonymousSpecError(f"Cannot concretize anonymous {type(self)} {self}") + + if not self.namespace: + # TODO interface combination ## + self._namespace = self.object_class.namespace + + # TODO interface combination ## + for name, variant in self.object_class.variants.items(): + if name not in self.variants: + self._variants[name] = variant.default + + for name, values in self.variants.items(): + if name not in self.object_class.variants: + raise Exception(f"{name} is not a valid variant of {self.name}") + + variant = self.object_class.variants[name] + variant.validate_values(self.variants[name]) + + # Convert to immutable type + self._variants = ConcreteVariantMap(self.variants) + + +class ConcreteExperimentSpec(ConcreteSpec, ExperimentSpec): + @property + def experiment(self) -> "benchpark.Experiment": + return self.experiment_class(self) + + +class SystemSpec(Spec): + @property + def system_class(self): + cls = sys_repo.get_obj_class(self.name) + # TODO: this shouldn't be necessary, but .template_dir isn't working + cls.resource_location = pathlib.Path( + sys_repo.filename_for_object_name(self.name) + ).parent + return cls + + @property + def object_class(self): + # shared getter so that multiple spec types can be concretized similarly + return self.system_class + + def concretize(self): + return ConcreteSystemSpec(self) + + +class ConcreteSystemSpec(ConcreteSpec, SystemSpec): + @property + def system(self) -> "benchpark.System": + return self.system_class(self) + + +# PARSING STUFF BELOW HERE + +#: Valid name for specs and variants. Here we are not using +#: the previous "w[\w.-]*" since that would match most +#: characters that can be part of a word in any language +IDENTIFIER = r"(?:[a-zA-Z_0-9][a-zA-Z_0-9\-]*)" +DOTTED_IDENTIFIER = rf"(?:{IDENTIFIER}(?:\.{IDENTIFIER})+)" +NAME = r"[a-zA-Z_0-9][a-zA-Z_0-9\-.]*" + +#: These are legal values that *can* be parsed bare, without quotes on the command line. +VALUE = r"(?:[a-zA-Z_0-9\-+\*.,:=\~\/\\]+)" + +#: Regex with groups to use for splitting (optionally propagated) key-value pairs +SPLIT_KVP = re.compile(rf"^({NAME})=(.*)$") + + +class TokenBase(enum.Enum): + """Base class for an enum type with a regex value""" + + def __new__(cls, *args, **kwargs): + # See + value = len(cls.__members__) + 1 + obj = object.__new__(cls) + obj._value_ = value + return obj + + def __init__(self, regex): + self.regex = regex + + def __str__(self): + return f"{self._name_}" + + +class TokenType(TokenBase): + """Enumeration of the different token kinds in the spec grammar. + + Order of declaration is extremely important, since text containing specs is parsed with a + single regex obtained by ``"|".join(...)`` of all the regex in the order of declaration. + """ + + # variants + BOOL_VARIANT = rf"(?:[~+-]\s*{NAME})" + KEY_VALUE_PAIR = rf"(?:{NAME}=(?:{VALUE}))" + # Package name + FULLY_QUALIFIED_PACKAGE_NAME = rf"(?:{DOTTED_IDENTIFIER})" + UNQUALIFIED_PACKAGE_NAME = rf"(?:{IDENTIFIER})" + # White spaces + WS = r"(?:\s+)" + + +class ErrorTokenType(TokenBase): + """Enum with regexes for error analysis""" + + # Unexpected character + UNEXPECTED = r"(?:.[\s]*)" + + +class Token: + """Represents tokens; generated from input by lexer and fed to parse().""" + + __slots__ = "kind", "value", "start", "end" + + def __init__( + self, + kind: TokenBase, + value: str, + start: Optional[int] = None, + end: Optional[int] = None, + ): + self.kind = kind + self.value = value + self.start = start + self.end = end + + def __repr__(self): + return str(self) + + def __str__(self): + return f"({self.kind}, {self.value})" + + def __eq__(self, other): + return (self.kind == other.kind) and (self.value == other.value) + + +#: List of all the regexes used to match spec parts, in order of precedence +TOKEN_REGEXES = [rf"(?P<{token}>{token.regex})" for token in TokenType] +#: List of all valid regexes followed by error analysis regexes +ERROR_HANDLING_REGEXES = TOKEN_REGEXES + [ + rf"(?P<{token}>{token.regex})" for token in ErrorTokenType +] +#: Regex to scan a valid text +ALL_TOKENS = re.compile("|".join(TOKEN_REGEXES)) +#: Regex to analyze an invalid text +ANALYSIS_REGEX = re.compile("|".join(ERROR_HANDLING_REGEXES)) + + +def tokenize(text: str) -> Iterable[Token]: + """Return a token generator from the text passed as input. + + Raises: + SpecTokenizationError: if we can't tokenize anymore, but didn't reach the + end of the input text. + """ + scanner = ALL_TOKENS.scanner(text) # type: ignore[attr-defined] + match: Optional[Match] = None + for match in iter(scanner.match, None): + # The following two assertions are to help mypy + msg = "unexpected value encountered during parsing. Please submit a bug report " + assert match is not None, msg + assert match.lastgroup is not None, msg + yield Token( + TokenType.__members__[match.lastgroup], + match.group(), + match.start(), + match.end(), + ) + + if match is None and not text: + # We just got an empty string + return + + if match is None or match.end() != len(text): + error_scanner = ANALYSIS_REGEX.scanner(text) + matches = [m for m in iter(error_scanner.match, None)] + raise SpecTokenizationError(matches, text) + + +class TokenContext: + """Token context passed around by parsers""" + + __slots__ = "token_stream", "current_token", "next_token" + + def __init__(self, token_stream: Iterator[Token]): + self.token_stream = token_stream + self.current_token = None + self.next_token = None + self.advance() + + def advance(self): + """Advance one token""" + self.current_token, self.next_token = self.next_token, next( + self.token_stream, None + ) + + def accept(self, kind: TokenType): + """If the next token is of the specified kind, advance the stream and return True. + Otherwise return False. + """ + if self.next_token and self.next_token.kind == kind: + self.advance() + return True + return False + + def expect(self, *kinds: TokenType): + return self.next_token and self.next_token.kind in kinds + + +class SpecParser(object): + __slots__ = "literal_str", "ctx", "type" + + def __init__(self, type, literal_str: str): + self.literal_str = literal_str + self.ctx = TokenContext( + filter(lambda x: x.kind != TokenType.WS, tokenize(literal_str)) + ) + self.type = type + + def tokens(self) -> List[Token]: + """Return the entire list of token from the initial text. White spaces are + filtered out. + """ + return list( + filter(lambda x: x.kind != TokenType.WS, tokenize(self.literal_str)) + ) + + def next_spec(self) -> Optional[Spec]: + """Return the next spec parsed from text. + + Args: + initial_spec: object where to parse the spec. If None a new one + will be created. + + Return + The spec that was parsed + """ + if not self.ctx.next_token: + return None + + spec = self.type() + + if self.ctx.accept(TokenType.UNQUALIFIED_PACKAGE_NAME): + spec.name = self.ctx.current_token.value + elif self.ctx.accept(TokenType.FULLY_QUALIFIED_PACKAGE_NAME): + parts = self.ctx.current_token.value.split(".") + name = parts[-1] + namespace = ".".join(parts[:-1]) + spec.name = name + spec.namespace = namespace + + while True: + if self.ctx.accept(TokenType.BOOL_VARIANT): + name = self.ctx.current_token.value[1:].strip() + value = self.ctx.current_token.value[0] == "+" + spec.variants[name] = str(value).lower() + elif self.ctx.accept(TokenType.KEY_VALUE_PAIR): + match = SPLIT_KVP.match(self.ctx.current_token.value) + assert ( + match + ), f"SPLIT_KVP cannot split pair {self.ctx.current_token.value}" + + name, value = match.groups() + spec.variants[name] = value + else: + break + + return spec + + def all_specs(self) -> List[Spec]: + return list(iter(self.next_spec, None)) + + +# ERROR HANDLING BELOW HERE + + +class AnonymousSpecError(Exception): + pass + + +class SpecTokenizationError(Exception): + """Syntax error in a spec string""" + + def __init__(self, matches, text): + message = "unexpected tokens in the spec string\n" + message += f"{text}" + + underline = "\n" + for match in matches: + if match.lastgroup == str(ErrorTokenType.UNEXPECTED): + underline += f"{'^' * (match.end() - match.start())}" + continue + underline += f"{' ' * (match.end() - match.start())}" + + message += underline + super().__init__(message) diff --git a/lib/benchpark/system.py b/lib/benchpark/system.py new file mode 100644 index 000000000..c220f628f --- /dev/null +++ b/lib/benchpark/system.py @@ -0,0 +1,172 @@ +# SPDX-License-Identifier: Apache-2.0 + +import hashlib +import importlib.util +import os +import pathlib +import sys + +import benchpark.paths +from benchpark.directives import ExperimentSystemBase +import benchpark.repo +from benchpark.runtime import RuntimeResources + +from typing import Dict, Tuple +import benchpark.spec +import benchpark.variant + +bootstrapper = RuntimeResources(benchpark.paths.benchpark_home) # noqa +bootstrapper.bootstrap() # noqa + +import ramble.config as cfg # noqa +import ramble.language.language_helpers # noqa +import ramble.language.shared_language # noqa +import spack.util.spack_yaml as syaml # noqa + +# We cannot import this the normal way because it from modern Spack +# and mixing modern Spack modules with ramble modules that depend on +# ancient Spack will cause errors. This module is safe to load as an +# individual because it is not used by Ramble +# The following code block implements the line +# import spack.schema.packages as packages_schema +schemas = { + "spack.schema.packages": f"{bootstrapper.spack_location}/lib/spack/spack/schema/packages.py", + "spack.schema.compilers": f"{bootstrapper.spack_location}/lib/spack/spack/schema/compilers.py", +} + + +def load_schema(schema_id, schema_path): + schema_spec = importlib.util.spec_from_file_location(schema_id, schema_path) + schema = importlib.util.module_from_spec(schema_spec) + sys.modules[schema_id] = schema + schema_spec.loader.exec_module(schema) + return schema + + +packages_schema = load_schema( + "spack.schema.packages", + f"{bootstrapper.spack_location}/lib/spack/spack/schema/packages.py", +) +compilers_schema = load_schema( + "spack.schema.compilers", + f"{bootstrapper.spack_location}/lib/spack/spack/schema/compilers.py", +) + + +_repo_path = benchpark.repo.paths[benchpark.repo.ObjectTypes.systems] + + +def _hash_id(content_list): + sha256_hash = hashlib.sha256() + for x in content_list: + sha256_hash.update(x.encode("utf-8")) + return sha256_hash.hexdigest() + + +class System(ExperimentSystemBase): + variants: Dict[ + str, + Tuple["benchpark.variant.Variant", "benchpark.spec.ConcreteSystemSpec"], + ] + + def __init__(self, spec): + self.spec: "benchpark.spec.ConcreteSystemSpec" = spec + super().__init__() + + def initialize(self): + self.external_resources = None + + self.sys_cores_per_node = None + self.sys_gpus_per_node = None + self.sys_mem_per_node = None + self.scheduler = None + self.timeout = "120" + self.queue = None + + self.required = ["sys_cores_per_node", "scheduler", "timeout"] + + def generate_description(self, output_dir): + self.initialize() + output_dir = pathlib.Path(output_dir) + + variables_yaml = output_dir / "variables.yaml" + with open(variables_yaml, "w") as f: + f.write(self.variables_yaml()) + + self.external_packages(output_dir) + self.compiler_description(output_dir) + + system_id_path = output_dir / "system_id.yaml" + with open(system_id_path, "w") as f: + f.write( + f"""\ +system: + name: {self.__class__.__name__} +""" + ) + + def system_id(self): + return _hash_id([self.variables_yaml()]) + + def _merge_config_files(self, schema, selections, dst_path): + data = cfg.read_config_file(selections[0], schema) + for selection in selections[1:]: + cfg.merge_yaml(data, cfg.read_config_file(selection, schema)) + + with open(dst_path, "w") as outstream: + syaml.dump_config(data, outstream) + + def external_pkg_configs(self): + return None + + def compiler_configs(self): + return None + + def external_packages(self, output_dir): + selections = self.external_pkg_configs() + if not selections: + return + + aux = output_dir / "auxiliary_software_files" + os.makedirs(aux, exist_ok=True) + aux_packages = aux / "packages.yaml" + + self._merge_config_files(packages_schema.schema, selections, aux_packages) + + def compiler_description(self, output_dir): + selections = self.compiler_configs() + if not selections: + return + + aux = output_dir / "auxiliary_software_files" + os.makedirs(aux, exist_ok=True) + aux_compilers = aux / "compilers.yaml" + + self._merge_config_files(compilers_schema.schema, selections, aux_compilers) + + def variables_yaml(self): + for attr in self.required: + if not getattr(self, attr, None): + raise ValueError(f"Missing required info: {attr}") + + optionals = list() + for opt in ["sys_gpus_per_node", "sys_mem_per_node", "queue"]: + if getattr(self, opt, None): + optionals.append(f"{opt}: {getattr(self, opt)}") + indent = " " * 2 + if optionals: + optionals_as_cfg = f"\n{indent}".join(optionals) + return f"""\ +# SPDX-License-Identifier: Apache-2.0 + +variables: + timeout: "{self.timeout}" + scheduler: "{self.scheduler}" + sys_cores_per_node: "{self.sys_cores_per_node}" + {optionals_as_cfg} + max_request: "1000" # n_ranks/n_nodes cannot exceed this + n_ranks: '1000001' # placeholder value + n_nodes: '1000001' # placeholder value + batch_submit: "placeholder" + mpi_command: "placeholder" +""" diff --git a/lib/benchpark/variant.py b/lib/benchpark/variant.py new file mode 100644 index 000000000..06e765138 --- /dev/null +++ b/lib/benchpark/variant.py @@ -0,0 +1,118 @@ +import inspect + + +class Variant: + """Represents a variant in a package, as declared in the + variant directive. + """ + + def __init__( + self, + name, + default, + description, + values=(True, False), + multi=False, + validator=None, + sticky=False, + ): + """Initialize a package variant. + + Args: + name (str): name of the variant + default (str): default value for the variant in case + nothing has been specified + description (str): purpose of the variant + values (sequence): sequence of allowed values or a callable + accepting a single value as argument and returning True if the + value is good, False otherwise + multi (bool): whether multiple CSV are allowed + validator (callable): optional callable used to enforce + additional logic on the set of values being validated + sticky (bool): if true the variant is set to the default value at + concretization time + """ + self.name = name + self.default = default + self.description = str(description) + + self.values = None + if values == "*": + # wildcard is a special case to make it easy to say any value is ok + self.validator = lambda x: True + + elif isinstance(values, type): + # supplying a type means any value *of that type* + def isa_type(v): + try: + values(v) + return True + except ValueError: + return False + + self.validator = isa_type + + elif callable(values): + # If 'values' is a callable, assume it is a single value + # validator and reset the values to be explicit during debug + self.validator = values + else: + # Otherwise, assume values is the set of allowed explicit values + self.values = tuple(values) + self.validator = lambda x: x in self.values + + self.multi = multi + self.sticky = sticky + + def validate_values(self, variant_values, pkg_cls=None): + """Validate a variant spec against this package variant. Raises an + exception if any error is found. + + Args: + vspec_values (tuple): values to be validated + pkg_cls (spack.package_base.PackageBase): the package class + that required the validation, if available + + Raises: Exception + """ + # If the value is exclusive there must be at most one + if not self.multi and len(variant_values) != 1: + raise Exception() + + # Check and record the values that are not allowed + not_allowed_values = [ + x for x in variant_values if x != "*" and self.validator(x) is False + ] + if not_allowed_values: + raise ValueError(f"{not_allowed_values} are not valid values for {pkg_cls}") + + @property + def allowed_values(self): + """Returns a string representation of the allowed values for + printing purposes + + Returns: + str: representation of the allowed values + """ + # Join an explicit set of allowed values + if self.values is not None: + v = tuple(str(x) for x in self.values) + return ", ".join(v) + # In case we were given a single-value validator + # print the docstring + docstring = inspect.getdoc(self.single_value_validator) + v = docstring if docstring else "" + return v + + def __eq__(self, other): + return ( + self.name == other.name + and self.default == other.default + and self.values == other.values + and self.multi == other.multi + and self.single_value_validator == other.single_value_validator + and self.group_validator == other.group_validator + ) + + def __ne__(self, other): + return not self == other diff --git a/lib/main.py b/lib/main.py new file mode 100755 index 000000000..5965101e2 --- /dev/null +++ b/lib/main.py @@ -0,0 +1,527 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import os +import pathlib +import shlex +import shutil +import subprocess +import sys +import yaml + +import benchpark.cmd.system +from benchpark.runtime import RuntimeResources + +DEBUG = False + +__version__ = "0.1.0" + + +def debug_print(message): + if DEBUG: + print("(debug) " + str(message)) + + +def main(): + if sys.version_info[:2] < (3, 8): + raise Exception("Benchpark requires at least python 3.8+.") + + parser = argparse.ArgumentParser(description="Benchpark") + parser.add_argument( + "-V", "--version", action="store_true", help="show version number and exit" + ) + + subparsers = parser.add_subparsers(title="Subcommands", dest="subcommand") + + actions = {} + benchpark_list(subparsers, actions) + benchpark_setup(subparsers, actions) + benchpark_tags(subparsers, actions) + init_commands(subparsers, actions) + + args = parser.parse_args() + no_args = True if len(sys.argv) == 1 else False + + if no_args: + parser.print_help() + return 1 + + if args.version: + print(get_version()) + return 0 + + if args.subcommand in actions: + actions[args.subcommand](args) + else: + print( + "Invalid subcommand ({args.subcommand}) - must choose one of: " + + " ".join(actions.keys()) + ) + + +def get_version(): + benchpark_version = __version__ + return benchpark_version + + +def source_location(): + script_location = os.path.dirname(os.path.abspath(__file__)) + return pathlib.Path(script_location).parent + + +def benchpark_list(subparsers, actions_dict): + list_parser = subparsers.add_parser( + "list", help="List available experiments, systems, and modifiers" + ) + list_parser.add_argument("sublist", nargs="?") + actions_dict["list"] = benchpark_list_handler + + +def benchpark_benchmarks(): + source_dir = source_location() + benchmarks = [] + experiments_dir = source_dir / "experiments" + for x in os.listdir(experiments_dir): + benchmarks.append(f"{x}") + return benchmarks + + +def benchpark_experiments(): + source_dir = source_location() + experiments = [] + experiments_dir = source_dir / "experiments" + for x in os.listdir(experiments_dir): + for y in os.listdir(experiments_dir / x): + experiments.append(f"{x}/{y}") + return experiments + + +def benchpark_systems(): + source_dir = source_location() + systems = [] + for x in os.listdir(source_dir / "configs"): + if not ( + os.path.isfile(os.path.join(source_dir / "configs", x)) or x == "common" + ): + systems.append(x) + return systems + + +def benchpark_modifiers(): + source_dir = source_location() + modifiers = [] + for x in os.listdir(source_dir / "modifiers"): + modifiers.append(x) + return modifiers + + +def benchpark_get_tags(): + f = source_location() / "tags.yaml" + tags = [] + + with open(f, "r") as stream: + try: + data = yaml.safe_load(stream) + except yaml.YAMLError as exc: + print(exc) + + for k0, v0 in data.items(): + if k0 == "benchpark-tags": + for k, v in v0.items(): + if isinstance(v, list): + for i in v: + tags.append(i) + else: + print("ERROR file does not contain benchpark-tags") + + return tags + + +def benchpark_list_handler(args): + sublist = args.sublist + benchmarks = benchpark_benchmarks() + experiments = benchpark_experiments() + systems = benchpark_systems() + modifiers = benchpark_modifiers() + + if sublist is None: + print("Experiments:") + for experiment in experiments: + print(f"\t{experiment}") + print("Systems:") + for system in systems: + print(f"\t{system}") + elif sublist == "benchmarks": + print("Benchmarks:") + for benchmark in benchmarks: + print(f"\t{benchmark}") + elif sublist == "experiments": + print("Experiments:") + for experiment in experiments: + print(f"\t{experiment}") + elif sublist == "systems": + print("Systems:") + for system in systems: + print(f"\t{system}") + elif sublist == "modifiers": + print("Modifiers:") + for modifier in modifiers: + print(f"\t{modifier}") + else: + raise ValueError( + f'Invalid benchpark list "{sublist}" - must choose [experiments], [systems], [modifiers] or leave empty' + ) + + +def benchpark_check_benchmark(arg_str): + benchmarks = benchpark_benchmarks() + found = arg_str in benchmarks + if not found: + out_str = f'Invalid benchmark "{arg_str}" - must choose one of: ' + for benchmark in benchmarks: + out_str += f"\n\t{benchmark}" + raise ValueError(out_str) + return found + + +def benchpark_check_experiment(arg_str): + experiments = benchpark_experiments() + found = arg_str in experiments + if not found: + out_str = f'Invalid experiment (benchmark/ProgrammingModel) "{arg_str}" - must choose one of: ' + for experiment in experiments: + out_str += f"\n\t{experiment}" + raise ValueError(out_str) + return found + + +def benchpark_check_system(arg_str): + # First check if it's a directory that contains a system_id.yaml + cfg_path = pathlib.Path(arg_str) + if cfg_path.is_dir(): + system_id_path = cfg_path / "system_id.yaml" + if system_id_path.exists(): + with open(system_id_path, "r") as f: + data = yaml.safe_load(f) + system_id = data["system"]["name"] + return system_id, cfg_path + + # If it's not a directory, it might be a shorthand that refers + # to a pre-constructed config + systems = benchpark_systems() + if arg_str not in systems: + out_str = ( + f"Invalid system {arg_str}: must choose one of:" + "\n\t(a) A system ID from `benchpark systems`" + "\n\t(b) A directory containing system_id.yaml" + ) + raise ValueError(out_str) + + configs_src_dir = source_location() / "configs" / str(arg_str) + return arg_str, configs_src_dir + + +def benchpark_check_tag(arg_str): + tags = benchpark_get_tags() + found = arg_str in tags + if not found: + out_str = f'Invalid tag "{arg_str}" - must choose one of: ' + for tag in tags: + out_str += f"\n\t{tag}" + raise ValueError(out_str) + return found + + +def benchpark_check_modifier(arg_str): + modifiers = benchpark_modifiers() + found = arg_str in modifiers + if not found: + out_str = f'Invalid modifier "{arg_str}" - must choose one of: ' + for modifier in modifiers: + out_str += f"\n\t{modifier}" + raise ValueError(out_str) + return found + + +def benchpark_setup(subparsers, actions_dict): + create_parser = subparsers.add_parser( + "setup", help="Set up an experiment and prepare it to build/run" + ) + + create_parser.add_argument( + "experiment", + type=str, + help="The experiment (benchmark/ProgrammingModel) to run", + ) + create_parser.add_argument( + "system", type=str, help="The system on which to run the experiment" + ) + create_parser.add_argument( + "experiments_root", + type=str, + help="Where to install packages and store results for the experiments. Benchpark expects to manage this directory, and it should be empty/nonexistent the first time you run benchpark setup experiments.", + ) + create_parser.add_argument( + "--modifier", + type=str, + default="none", + help="The modifier to apply to the experiment (default none)", + ) + + actions_dict["setup"] = benchpark_setup_handler + + +def init_commands(subparsers, actions_dict): + """This function is for initializing commands that are defined outside + of this script. It is intended that all command setup will eventually + be refactored in this way (e.g. `benchpark_setup` will be defined in + another file. + """ + system_parser = subparsers.add_parser("system", help="Initialize a system config") + benchpark.cmd.system.setup_parser(system_parser) + actions_dict["system"] = benchpark.cmd.system.command + + +def run_command(command_str, env=None): + proc = subprocess.Popen( + shlex.split(command_str), + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + stdout, stderr = proc.communicate() + if proc.returncode != 0: + raise RuntimeError( + f"Failed command: {command_str}\nOutput: {stdout}\nError: {stderr}" + ) + + return (stdout, stderr) + + +def benchpark_tags(subparsers, actions_dict): + create_parser = subparsers.add_parser("tags", help="Tags in Benchpark experiments") + create_parser.add_argument( + "experiments_root", + type=str, + help="The experiments_root you specified during Benchpark setup.", + ) + create_parser.add_argument( + "-a", + "--application", + action="store", + help="The application for which to find Benchpark tags", + ) + create_parser.add_argument( + "-t", + "--tag", + action="store", + help="The tag for which to search in Benchpark experiments", + ) + actions_dict["tags"] = benchpark_tags_handler + + +# Note: it would be nice to vendor spack.llnl.util.link_tree, but that +# involves pulling in most of llnl/util/ and spack/util/ +def symlink_tree(src, dst, include_fn=None): + """Like ``cp -R`` but instead of files, create symlinks""" + src = os.path.abspath(src) + dst = os.path.abspath(dst) + # By default, we include all filenames + include_fn = include_fn or (lambda f: True) + for x in [src, dst]: + if not os.path.isdir(x): + raise ValueError(f"Not a directory: {x}") + for src_subdir, directories, files in os.walk(src): + relative_src_dir = pathlib.Path(os.path.relpath(src_subdir, src)) + dst_dir = pathlib.Path(dst) / relative_src_dir + dst_dir.mkdir(parents=True, exist_ok=True) + for x in files: + if not include_fn(x): + continue + dst_symlink = dst_dir / x + src_file = os.path.join(src_subdir, x) + os.symlink(src_file, dst_symlink) + + +def benchpark_setup_handler(args): + """ + experiments_root/ + spack/ + ramble/ + / + / + workspace/ + configs/ + (everything from source/configs/) + (everything from source/experiments/) + """ + + experiment = args.experiment + system = args.system + experiments_root = pathlib.Path(os.path.abspath(args.experiments_root)) + modifier = args.modifier + source_dir = source_location() + debug_print(f"source_dir = {source_dir}") + debug_print(f"specified experiment (benchmark/ProgrammingModel) = {experiment}") + benchpark_check_experiment(experiment) + debug_print(f"specified system = {system}") + system_id, configs_src_dir = benchpark_check_system(system) + debug_print(f"specified modifier = {modifier}") + benchpark_check_modifier(modifier) + + workspace_dir = experiments_root / str(experiment) / str(system_id) + + if workspace_dir.exists(): + if workspace_dir.is_dir(): + print(f"Clearing existing workspace {workspace_dir}") + shutil.rmtree(workspace_dir) + else: + print( + f"Benchpark expects to manage {workspace_dir} as a directory, but it is not" + ) + sys.exit(1) + + workspace_dir.mkdir(parents=True) + + ramble_workspace_dir = workspace_dir / "workspace" + ramble_configs_dir = ramble_workspace_dir / "configs" + ramble_logs_dir = ramble_workspace_dir / "logs" + ramble_spack_experiment_configs_dir = ( + ramble_configs_dir / "auxiliary_software_files" + ) + + print(f"Setting up configs for Ramble workspace {ramble_configs_dir}") + + experiment_src_dir = source_dir / "experiments" / experiment + modifier_config_dir = source_dir / "modifiers" / modifier / "configs" + ramble_configs_dir.mkdir(parents=True) + ramble_logs_dir.mkdir(parents=True) + ramble_spack_experiment_configs_dir.mkdir(parents=True) + + def include_fn(fname): + # Only include .yaml and .tpl files + # Always exclude files that start with "." + if fname.startswith("."): + return False + if fname.endswith(".yaml"): + return True + return False + + symlink_tree(configs_src_dir, ramble_configs_dir, include_fn) + symlink_tree(experiment_src_dir, ramble_configs_dir, include_fn) + symlink_tree(modifier_config_dir, ramble_configs_dir, include_fn) + symlink_tree( + source_dir / "configs" / "common", + ramble_spack_experiment_configs_dir, + include_fn, + ) + + template_name = "execute_experiment.tpl" + experiment_template_options = [ + configs_src_dir / template_name, + experiment_src_dir / template_name, + source_dir / "common-resources" / template_name, + ] + for choice_template in experiment_template_options: + if os.path.exists(choice_template): + break + os.symlink( + choice_template, + ramble_configs_dir / "execute_experiment.tpl", + ) + + initializer_script = experiments_root / "setup.sh" + + per_workspace_setup = RuntimeResources(experiments_root) + + spack, first_time_spack = per_workspace_setup.spack_first_time_setup() + ramble, first_time_ramble = per_workspace_setup.ramble_first_time_setup() + + if first_time_spack: + spack("repo", "add", "--scope=site", f"{source_dir}/repo") + + if first_time_ramble: + ramble(f"repo add --scope=site {source_dir}/repo") + ramble('config --scope=site add "config:disable_progress_bar:true"') + ramble(f"repo add -t modifiers --scope=site {source_dir}/modifiers") + ramble("config --scope=site add \"config:spack:global:args:'-d'\"") + + if not initializer_script.exists(): + with open(initializer_script, "w") as f: + f.write( + f"""\ +if [ -n "${{_BENCHPARK_INITIALIZED:-}}" ]; then + return 0 +fi + +. {per_workspace_setup.spack_location}/share/spack/setup-env.sh +. {per_workspace_setup.ramble_location}/share/ramble/setup-env.sh + +export SPACK_DISABLE_LOCAL_CONFIG=1 + +export _BENCHPARK_INITIALIZED=true +""" + ) + + instructions = f"""\ +To complete the benchpark setup, do the following: + + . {initializer_script} + +Further steps are needed to build the experiments (ramble -P -D {ramble_workspace_dir} workspace setup) and run them (ramble -P -D {ramble_workspace_dir} on) +""" + print(instructions) + + +def helper_experiments_tags(ramble_exe, benchmarks): + # find all tags in Ramble applications (both in Ramble built-in and in Benchpark/repo) + (tags_stdout, tags_stderr) = run_command(f"{ramble_exe} attributes --tags --all") + ramble_applications_tags = {} + lines = tags_stdout.splitlines() + + for line in lines: + key_value = line.split(":") + ramble_applications_tags[key_value[0]] = key_value[1].strip().split(",") + + benchpark_experiments_tags = {} + for benchmark in benchmarks: + benchpark_experiments_tags[benchmark] = ramble_applications_tags[benchmark] + return benchpark_experiments_tags + + +def benchpark_tags_handler(args): + """ + Filter ramble tags by benchpark benchmarks + """ + experiments_root = pathlib.Path(os.path.abspath(args.experiments_root)) + ramble_location = experiments_root / "ramble" + ramble_exe = ramble_location / "bin" / "ramble" + benchmarks = benchpark_benchmarks() + + if args.tag: + if benchpark_check_tag(args.tag): + # find all applications in Ramble that have a given tag (both in Ramble built-in and in Benchpark/repo) + (tag_stdout, tag_stderr) = run_command(f"{ramble_exe} list -t {args.tag}") + lines = tag_stdout.splitlines() + + for line in lines: + if line in benchmarks: + print(line) + + elif args.application: + if benchpark_check_benchmark(args.application): + benchpark_experiments_tags = helper_experiments_tags(ramble_exe, benchmarks) + print(benchpark_experiments_tags[args.application]) + else: + benchpark_experiments_tags = helper_experiments_tags(ramble_exe, benchmarks) + print("All tags that exist in Benchpark experiments:") + for k, v in benchpark_experiments_tags.items(): + print(k) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index bc919dd96..6b6bc0dd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ requires-python = ">=3.8" line-length = 88 color = true target-version = ['py38', 'py39', 'py310', 'py311'] -include = '\.pyi?$|bin\/benchpark' +include = '\.pyi?$|bin\/benchpark$' exclude = ''' /( \.eggs @@ -24,11 +24,37 @@ exclude = ''' | repo )/ ''' +force-exclude = ''' +/( + '/lib/benchpark/test_repo/(/.*)?$' + | /lib/benchpark/test_repo(/.*)?$ + | lib/benchpark/test_repo + | '/bin/benchpark-python' + | \.github +)/ +''' [tool.isort] profile = "black" skip_gitignore = true color_output = true +skip_glob = [ + "lib/benchpark/test_repo/**" +] + +[tool.flake8] +exclude = [ + "lib/benchpark/test_repo/**" +] +skip_glob = [ + "lib/benchpark/test_repo/**" +] +force-exclude = [ + "./lib/benchpark/test_repo/**" +] +per-file-ignores = """ + ./lib/benchpark/test_repo/** +""" [tool.codespell] skip = './docs/_build,./docs/_static' diff --git a/var/exp_repo/experiments/saxpy/experiment.py b/var/exp_repo/experiments/saxpy/experiment.py new file mode 100644 index 000000000..144615bf5 --- /dev/null +++ b/var/exp_repo/experiments/saxpy/experiment.py @@ -0,0 +1,68 @@ +from benchpark.directives import variant +from benchpark.experiment import Experiment + + +class Saxpy(Experiment): + variant( + "programming_model", + default="openmp", + values=("openmp", "cuda", "rocm"), + description="on-node parallelism model", + ) + + def compute_applications_section(self): + variables = {} + matrix = {} + + # GPU tests include some smaller sizes + n = ["512", "1024"] + matrix = ["size"] + if self.spec.satisfies("programming_model=openmp"): + matrix += ["omp_num_threads"] + variables["n_nodes"] = ["1", "2"] + variables["n_ranks"] = "8" + variables["omp_num_threads"] = ["2", "4"] + else: + n = ["128", "256"] + n + variables["n_gpus"] = "1" + + variables["n"] = n + + return { + "saxpy": { # ramble Application name + # TODO replace with a hash once we have one? + f"problem-{str(self.spec)}": { + "experiments": { + "saxpy_{n}_{n_nodes}_{omp_num_threads}": { + "variables": variables, + "matrices": matrix, + } + } + } + } + } + + def compute_spack_section(self): + # TODO: express that we need certain variables from system + # Does not need to happen before merge, separate task + saxpy_spack_spec = "saxpy@1.0.0{modifier_spack_variant}" + if self.spec.satisfies("programming_model=openmp"): + saxpy_spack_spec += "+openmp" + elif self.spec.satisfies("programming_model=cuda"): + saxpy_spack_spec += "+cuda cuda_arch={cuda_arch}" + elif self.spec.satisfies("programming_model=rocm"): + saxpy_spack_spec += "+rocm amdgpu_target={rocm_arch}" + + packages = ["default-mpi", self.spec.name, "{modifier_package_name}"] + + return { + "spack": { + "packages": { + "saxpy": { + "spack_spec": saxpy_spack_spec, + "compiler": "default_compiler", # TODO: this should probably move? + } + }, + "environments": {"saxpy": {"packages": packages}}, + } + } diff --git a/var/exp_repo/repo.yaml b/var/exp_repo/repo.yaml new file mode 100644 index 000000000..54b282db6 --- /dev/null +++ b/var/exp_repo/repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: builtin diff --git a/var/sys_repo/repo.yaml b/var/sys_repo/repo.yaml new file mode 100644 index 000000000..4bd5dcb32 --- /dev/null +++ b/var/sys_repo/repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: sysbuiltin diff --git a/var/sys_repo/systems/aws/system.py b/var/sys_repo/systems/aws/system.py new file mode 100644 index 000000000..7d6a27ffb --- /dev/null +++ b/var/sys_repo/systems/aws/system.py @@ -0,0 +1,37 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +from benchpark.system import System +from benchpark.directives import variant + +# Taken from https://aws.amazon.com/ec2/instance-types/ +# With boto3, we could determine this dynamically vs. storing a static table +id_to_resources = { + "c4.xlarge": { + "sys_cores_per_node": 4, + "sys_mem_per_node": 7.5, + }, + "c6g.xlarge": { + "sys_cores_per_node": 4, + "sys_mem_per_node": 8, + }, +} + + +class Aws(System): + variant( + "instance_type", + values=("c6g.xlarge", "c4.xlarge"), + default="c4.xlarge", + description="AWS instance type", + ) + + def initialize(self): + super().initialize() + self.scheduler = "mpi" + # TODO: for some reason I have to index to get value, even if multi=False + attrs = id_to_resources.get(self.spec.variants["instance_type"][0]) + for k, v in attrs.items(): + setattr(self, k, v) diff --git a/var/sys_repo/systems/tioga/compilers/gcc/00-gcc-12-compilers.yaml b/var/sys_repo/systems/tioga/compilers/gcc/00-gcc-12-compilers.yaml new file mode 100644 index 000000000..2b4ec4618 --- /dev/null +++ b/var/sys_repo/systems/tioga/compilers/gcc/00-gcc-12-compilers.yaml @@ -0,0 +1,14 @@ +compilers: +- compiler: + spec: gcc@12.2.0 + paths: + cc: /opt/cray/pe/gcc/12.2.0/bin/gcc + cxx: /opt/cray/pe/gcc/12.2.0/bin/g++ + f77: /opt/cray/pe/gcc/12.2.0/bin/gfortran + fc: /opt/cray/pe/gcc/12.2.0/bin/gfortran + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] diff --git a/var/sys_repo/systems/tioga/compilers/rocm/00-rocm-551-compilers.yaml b/var/sys_repo/systems/tioga/compilers/rocm/00-rocm-551-compilers.yaml new file mode 100644 index 000000000..0952924cb --- /dev/null +++ b/var/sys_repo/systems/tioga/compilers/rocm/00-rocm-551-compilers.yaml @@ -0,0 +1,49 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +compilers: +- compiler: + spec: cce@16.0.0-rocm5.5.1 + paths: + cc: /opt/cray/pe/cce/16.0.0/bin/craycc + cxx: /opt/cray/pe/cce/16.0.0/bin/crayCC + f77: /opt/cray/pe/cce/16.0.0/bin/crayftn + fc: /opt/cray/pe/cce/16.0.0/bin/crayftn + flags: + cflags: -g -O2 + cxxflags: -g -O2 -std=c++17 + fflags: -g -O2 -hnopattern + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + prepend_path: + LD_LIBRARY_PATH: /opt/cray/pe/cce/16.0.0/cce/x86_64/lib + extra_rpaths: [/opt/cray/pe/cce/16.0.0/cce/x86_64/lib/, /opt/cray/pe/gcc-libs/] +- compiler: + spec: rocmcc@5.5.1 + paths: + cc: /opt/rocm-5.5.1/bin/amdclang + cxx: /opt/rocm-5.5.1/bin/amdclang++ + f77: /opt/rocm-5.5.1/bin/amdflang + fc: /opt/rocm-5.5.1/bin/amdflang + flags: + cflags: -g -O2 + cxxflags: -g -O2 + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + set: + RFE_811452_DISABLE: '1' + append_path: + LD_LIBRARY_PATH: /opt/cray/pe/gcc-libs + prepend_path: + LD_LIBRARY_PATH: "/opt/cray/pe/cce/16.0.0/cce/x86_64/lib:/opt/cray/pe/pmi/6.1.12/lib" + LIBRARY_PATH: /opt/rocm-5.5.1/lib + extra_rpaths: + - /opt/rocm-5.5.1/lib + - /opt/cray/pe/gcc-libs + - /opt/cray/pe/cce/16.0.0/cce/x86_64/lib diff --git a/var/sys_repo/systems/tioga/compilers/rocm/01-rocm-543-compilers.yaml b/var/sys_repo/systems/tioga/compilers/rocm/01-rocm-543-compilers.yaml new file mode 100644 index 000000000..a032c69ad --- /dev/null +++ b/var/sys_repo/systems/tioga/compilers/rocm/01-rocm-543-compilers.yaml @@ -0,0 +1,26 @@ +compilers: +- compiler: + spec: rocmcc@5.4.3 + paths: + cc: /opt/rocm-5.4.3/bin/amdclang + cxx: /opt/rocm-5.4.3/bin/amdclang++ + f77: /opt/rocm-5.4.3/bin/amdflang + fc: /opt/rocm-5.4.3/bin/amdflang + flags: + cflags: -g -O2 + cxxflags: -g -O2 + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + set: + RFE_811452_DISABLE: '1' + append_path: + LD_LIBRARY_PATH: /opt/cray/pe/gcc-libs + prepend_path: + LD_LIBRARY_PATH: "/opt/cray/pe/cce/16.0.0/cce/x86_64/lib:/opt/cray/pe/pmi/6.1.12/lib" + LIBRARY_PATH: /opt/rocm-5.4.3/lib + extra_rpaths: + - /opt/rocm-5.4.3/lib + - /opt/cray/pe/gcc-libs + - /opt/cray/pe/cce/16.0.0/cce/x86_64/lib diff --git a/var/sys_repo/systems/tioga/externals/base/00-packages.yaml b/var/sys_repo/systems/tioga/externals/base/00-packages.yaml new file mode 100644 index 000000000..567888887 --- /dev/null +++ b/var/sys_repo/systems/tioga/externals/base/00-packages.yaml @@ -0,0 +1,186 @@ +packages: + all: + require: 'target=x86_64:' + variants: amdgpu_target=gfx90a + tar: + externals: + - spec: tar@1.30 + prefix: /usr + coreutils: + externals: + - spec: coreutils@8.30 + prefix: /usr + libtool: + externals: + - spec: libtool@2.4.6 + prefix: /usr + flex: + externals: + - spec: flex@2.6.1+lex + prefix: /usr + openssl: + externals: + - spec: openssl@1.1.1k + prefix: /usr + m4: + externals: + - spec: m4@1.4.18 + prefix: /usr + groff: + externals: + - spec: groff@1.22.3 + prefix: /usr + cmake: + externals: + - spec: cmake@3.20.2 + prefix: /usr + - spec: cmake@3.23.1 + prefix: /usr/tce + - spec: cmake@3.24.2 + prefix: /usr/tce + buildable: false + pkgconf: + externals: + - spec: pkgconf@1.4.2 + prefix: /usr + curl: + externals: + - spec: curl@7.61.1+gssapi+ldap+nghttp2 + prefix: /usr + gmake: + externals: + - spec: gmake@4.2.1 + prefix: /usr + subversion: + externals: + - spec: subversion@1.10.2 + prefix: /usr + diffutils: + externals: + - spec: diffutils@3.6 + prefix: /usr + swig: + externals: + - spec: swig@3.0.12 + prefix: /usr + gawk: + externals: + - spec: gawk@4.2.1 + prefix: /usr + binutils: + externals: + - spec: binutils@2.30.113 + prefix: /usr + findutils: + externals: + - spec: findutils@4.6.0 + prefix: /usr + git-lfs: + externals: + - spec: git-lfs@2.11.0 + prefix: /usr/tce + ccache: + externals: + - spec: ccache@3.7.7 + prefix: /usr + automake: + externals: + - spec: automake@1.16.1 + prefix: /usr + cvs: + externals: + - spec: cvs@1.11.23 + prefix: /usr + git: + externals: + - spec: git@2.31.1+tcltk + prefix: /usr + - spec: git@2.29.1+tcltk + prefix: /usr/tce + openssh: + externals: + - spec: openssh@8.0p1 + prefix: /usr + autoconf: + externals: + - spec: autoconf@2.69 + prefix: /usr + texinfo: + externals: + - spec: texinfo@6.5 + prefix: /usr + bison: + externals: + - spec: bison@3.0.4 + prefix: /usr + python: + externals: + - spec: python@3.9.12 + prefix: /usr/tce/packages/python/python-3.9.12 + buildable: false + unzip: + buildable: false + externals: + - spec: unzip@6.0 + prefix: /usr + hypre: + variants: amdgpu_target=gfx90a + hwloc: + externals: + - spec: hwloc@2.9.1 + prefix: /usr + buildable: false + fftw: + buildable: false + intel-oneapi-mkl: + externals: + - spec: intel-oneapi-mkl@2023.2.0 + prefix: /opt/intel/oneapi + buildable: false + lapack: + buildable: false + blas: + buildable: false + mpi: + buildable: false + libfabric: + externals: + - spec: libfabric@2.1 + prefix: /opt/cray/libfabric/2.1 + buildable: false + hipfft: + buildable: false + rocfft: + buildable: false + rocprim: + buildable: false + rocrand: + buildable: false + rocsparse: + buildable: false + rocthrust: + buildable: false + hip: + buildable: false + hsa-rocr-dev: + buildable: false + comgr: + buildable: false + hipsparse: + buildable: false + hipblas: + buildable: false + hsakmt-roct: + buildable: false + roctracer-dev-api: + buildable: false + rocminfo: + buildable: false + llvm: + buildable: false + llvm-amdgpu: + buildable: false + rocblas: + buildable: false + rocsolver: + buildable: false diff --git a/var/sys_repo/systems/tioga/externals/libsci/00-gcc-packages.yaml b/var/sys_repo/systems/tioga/externals/libsci/00-gcc-packages.yaml new file mode 100644 index 000000000..06e717f26 --- /dev/null +++ b/var/sys_repo/systems/tioga/externals/libsci/00-gcc-packages.yaml @@ -0,0 +1,5 @@ +packages: + cray-libsci: + externals: + - spec: cray-libsci@23.05.1.4%gcc + prefix: /opt/cray/pe/libsci/23.05.1.4/gnu/10.3/x86_64/ diff --git a/var/sys_repo/systems/tioga/externals/libsci/01-cce-packages.yaml b/var/sys_repo/systems/tioga/externals/libsci/01-cce-packages.yaml new file mode 100644 index 000000000..1dbe196c7 --- /dev/null +++ b/var/sys_repo/systems/tioga/externals/libsci/01-cce-packages.yaml @@ -0,0 +1,5 @@ +packages: + cray-libsci: + externals: + - spec: cray-libsci@23.05.1.4%cce + prefix: /opt/cray/pe/libsci/23.05.1.4/cray/12.0/x86_64/ diff --git a/var/sys_repo/systems/tioga/externals/mpi/00-gcc-ngtl-packages.yaml b/var/sys_repo/systems/tioga/externals/mpi/00-gcc-ngtl-packages.yaml new file mode 100644 index 000000000..4b2314e6e --- /dev/null +++ b/var/sys_repo/systems/tioga/externals/mpi/00-gcc-ngtl-packages.yaml @@ -0,0 +1,8 @@ +packages: + cray-mpich: + externals: + - spec: cray-mpich@8.1.26%gcc@12.2.0 ~gtl +wrappers + prefix: /opt/cray/pe/mpich/8.1.26/ofi/gnu/10.3 + extra_attributes: + gtl_lib_path: /opt/cray/pe/mpich/8.1.26/gtl/lib + ldflags: "-L/opt/cray/pe/mpich/8.1.26/ofi/gnu/10.3/lib -lmpi -L/opt/cray/pe/mpich/8.1.26/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/8.1.26/gtl/lib" diff --git a/var/sys_repo/systems/tioga/externals/mpi/01-cce-ngtl-packages.yaml b/var/sys_repo/systems/tioga/externals/mpi/01-cce-ngtl-packages.yaml new file mode 100644 index 000000000..dc7b7b3cc --- /dev/null +++ b/var/sys_repo/systems/tioga/externals/mpi/01-cce-ngtl-packages.yaml @@ -0,0 +1,8 @@ +packages: + cray-mpich: + externals: + - spec: cray-mpich@8.1.26%cce@16.0.0 ~gtl +wrappers + prefix: /opt/cray/pe/mpich/8.1.26/ofi/crayclang/16.0 + extra_attributes: + gtl_lib_path: /opt/cray/pe/mpich/8.1.26/gtl/lib + ldflags: "-L/opt/cray/pe/mpich/8.1.26/ofi/crayclang/16.0/lib -lmpi -L/opt/cray/pe/mpich/8.1.26/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/8.1.26/gtl/lib" diff --git a/var/sys_repo/systems/tioga/externals/mpi/02-cce-ygtl-packages.yaml b/var/sys_repo/systems/tioga/externals/mpi/02-cce-ygtl-packages.yaml new file mode 100644 index 000000000..1289b84ef --- /dev/null +++ b/var/sys_repo/systems/tioga/externals/mpi/02-cce-ygtl-packages.yaml @@ -0,0 +1,10 @@ +packages: + cray-mpich: + externals: + - spec: cray-mpich@8.1.26%cce@16.0.0 +gtl +wrappers + prefix: /opt/cray/pe/mpich/8.1.26/ofi/crayclang/16.0 + extra_attributes: + gtl_cutoff_size: 4096 + fi_cxi_ats: 0 + gtl_lib_path: /opt/cray/pe/mpich/8.1.26/gtl/lib + ldflags: "-L/opt/cray/pe/mpich/8.1.26/ofi/crayclang/16.0/lib -lmpi -L/opt/cray/pe/mpich/8.1.26/gtl/lib -Wl,-rpath=/opt/cray/pe/mpich/8.1.26/gtl/lib -lmpi_gtl_hsa" diff --git a/var/sys_repo/systems/tioga/externals/rocm/00-version-543-packages.yaml b/var/sys_repo/systems/tioga/externals/rocm/00-version-543-packages.yaml new file mode 100644 index 000000000..e4dea4c97 --- /dev/null +++ b/var/sys_repo/systems/tioga/externals/rocm/00-version-543-packages.yaml @@ -0,0 +1,73 @@ +packages: + hipfft: + externals: + - spec: hipfft@5.4.3 + prefix: /opt/rocm-5.4.3 + rocfft: + externals: + - spec: rocfft@5.4.3 + prefix: /opt/rocm-5.4.3 + rocprim: + externals: + - spec: rocprim@5.4.3 + prefix: /opt/rocm-5.4.3 + rocrand: + externals: + - spec: rocrand@5.4.3 + prefix: /opt/rocm-5.4.3/hiprand + rocsparse: + externals: + - spec: rocsparse@5.4.3 + prefix: /opt/rocm-5.4.3 + rocthrust: + externals: + - spec: rocthrust@5.4.3 + prefix: /opt/rocm-5.4.3 + hip: + externals: + - spec: hip@5.4.3 + prefix: /opt/rocm-5.4.3 + hsa-rocr-dev: + externals: + - spec: hsa-rocr-dev@5.4.3 + prefix: /opt/rocm-5.4.3 + comgr: + externals: + - spec: comgr@5.4.3 + prefix: /opt/rocm-5.4.3/ + hipsparse: + externals: + - spec: hipsparse@5.4.3 + prefix: /opt/rocm-5.4.3 + hipblas: + externals: + - spec: hipblas@5.4.3 + prefix: /opt/rocm-5.4.3/ + hsakmt-roct: + externals: + - spec: hsakmt-roct@5.4.3 + prefix: /opt/rocm-5.4.3/ + roctracer-dev-api: + externals: + - spec: roctracer-dev-api@5.4.3 + prefix: /opt/rocm-5.4.3/ + rocminfo: + externals: + - spec: rocminfo@5.4.3 + prefix: /opt/rocm-5.4.3/ + llvm: + externals: + - spec: llvm@15.0.0-5.4.3 + prefix: /opt/rocm-5.4.3/llvm + llvm-amdgpu: + externals: + - spec: llvm-amdgpu@5.4.3 + prefix: /opt/rocm-5.4.3/llvm + rocblas: + externals: + - spec: rocblas@5.4.3 + prefix: /opt/rocm-5.4.3 + rocsolver: + externals: + - spec: rocsolver@5.4.3 + prefix: /opt/rocm-5.4.3 diff --git a/var/sys_repo/systems/tioga/externals/rocm/01-version-551-packages.yaml b/var/sys_repo/systems/tioga/externals/rocm/01-version-551-packages.yaml new file mode 100644 index 000000000..2d871d34a --- /dev/null +++ b/var/sys_repo/systems/tioga/externals/rocm/01-version-551-packages.yaml @@ -0,0 +1,91 @@ +packages: + hipfft: + buildable: false + externals: + - spec: hipfft@5.5.1 + prefix: /opt/rocm-5.5.1 + rocfft: + buildable: false + externals: + - spec: rocfft@5.5.1 + prefix: /opt/rocm-5.5.1 + rocprim: + buildable: false + externals: + - spec: rocprim@5.5.1 + prefix: /opt/rocm-5.5.1 + rocrand: + buildable: false + externals: + - spec: rocrand@5.5.1 + prefix: /opt/rocm-5.5.1/hiprand + rocsparse: + buildable: false + externals: + - spec: rocsparse@5.5.1 + prefix: /opt/rocm-5.5.1 + rocthrust: + buildable: false + externals: + - spec: rocthrust@5.5.1 + prefix: /opt/rocm-5.5.1 + hip: + buildable: false + externals: + - spec: hip@5.5.1 + prefix: /opt/rocm-5.5.1 + hsa-rocr-dev: + buildable: false + externals: + - spec: hsa-rocr-dev@5.5.1 + prefix: /opt/rocm-5.5.1 + comgr: + buildable: false + externals: + - spec: comgr@5.5.1 + prefix: /opt/rocm-5.5.1/ + hipsparse: + buildable: false + externals: + - spec: hipsparse@5.5.1 + prefix: /opt/rocm-5.5.1 + hipblas: + buildable: false + externals: + - spec: hipblas@5.5.1 + prefix: /opt/rocm-5.5.1/ + hsakmt-roct: + buildable: false + externals: + - spec: hsakmt-roct@5.5.1 + prefix: /opt/rocm-5.5.1/ + roctracer-dev-api: + buildable: false + externals: + - spec: roctracer-dev-api@5.5.1 + prefix: /opt/rocm-5.5.1/ + rocminfo: + buildable: false + externals: + - spec: rocminfo@5.5.1 + prefix: /opt/rocm-5.5.1/ + llvm: + buildable: false + externals: + - spec: llvm@16.0.0-5.5.1 + prefix: /opt/rocm-5.5.1/llvm + llvm-amdgpu: + buildable: false + externals: + - spec: llvm-amdgpu@5.5.1 + prefix: /opt/rocm-5.5.1/llvm + rocblas: + buildable: false + externals: + - spec: rocblas@5.5.1 + prefix: /opt/rocm-5.5.1 + rocsolver: + buildable: false + externals: + - spec: rocsolver@5.5.1 + prefix: /opt/rocm-5.5.1 diff --git a/var/sys_repo/systems/tioga/system.py b/var/sys_repo/systems/tioga/system.py new file mode 100644 index 000000000..53c4380df --- /dev/null +++ b/var/sys_repo/systems/tioga/system.py @@ -0,0 +1,130 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +import pathlib + +from benchpark.directives import variant +from benchpark.system import System + + +class Tioga(System): + variant( + "rocm", + default="551", + values=("543", "551"), + description="ROCm version", + ) + + variant( + "compiler", + default="cce", + values=("gcc", "cce"), + description="Which compiler to use", + ) + + variant( + "gtl", + default=False, + values=("true", "false"), + description="Use GTL-enabled MPI", + ) + + def initialize(self): + super().initialize() + + self.scheduler = "flux" + self.sys_cores_per_node = "64" + self.sys_gpus_per_node = "4" + + def generate_description(self, output_dir): + super().generate_description(output_dir) + + sw_description = pathlib.Path(output_dir) / "software.yaml" + + with open(sw_description, "w") as f: + f.write(self.sw_description()) + + def external_pkg_configs(self): + externals = Tioga.resource_location / "externals" + + rocm = self.spec.variants["rocm"][0] + gtl = self.spec.variants["gtl"][0] + compiler = self.spec.variants["compiler"][0] + + selections = [externals / "base" / "00-packages.yaml"] + if rocm == "543": + selections.append(externals / "rocm" / "00-version-543-packages.yaml") + elif rocm == "551": + selections.append(externals / "rocm" / "01-version-551-packages.yaml") + + if compiler == "cce": + if gtl == "true": + selections.append(externals / "mpi" / "02-cce-ygtl-packages.yaml") + else: + selections.append(externals / "mpi" / "01-cce-ngtl-packages.yaml") + selections.append(externals / "libsci" / "01-cce-packages.yaml") + elif compiler == "gcc": + selections.append(externals / "mpi" / "00-gcc-ngtl-packages.yaml") + selections.append(externals / "libsci" / "00-gcc-packages.yaml") + + return selections + + def compiler_configs(self): + compilers = Tioga.resource_location / "compilers" + + compiler = self.spec.variants["compiler"][0] + # rocm = self.spec.variants["rocm"][0] + + selections = [] + # TODO: I'm not actually sure what compiler mixing is desired, if any + # so I don't think the choices here make much sense, but this + # demonstrate how system spec variants can be used to choose what + # configuration to construct + if compiler == "cce": + selections.append(compilers / "rocm" / "00-rocm-551-compilers.yaml") + elif compiler == "gcc": + selections.append(compilers / "gcc" / "00-gcc-12-compilers.yaml") + + return selections + + def sw_description(self): + """This is somewhat vestigial: for the Tioga config that is committed + to the repo, multiple instances of mpi/compilers are stored and + and these variables were used to choose consistent dependencies. + The configs generated by this class should only ever have one + instance of MPI etc., so there is no need for that. The experiments + will fail if these variables are not defined though, so for now + they are still generated (but with more-generic values). + """ + return """\ +software: + packages: + default-compiler: + pkg_spec: cce + default-mpi: + pkg_spec: cray-mpich + compiler-rocm: + pkg_spec: cce + compiler-amdclang: + pkg_spec: clang + compiler-gcc: + pkg_spec: gcc + blas-rocm: + pkg_spec: rocblas + blas: + pkg_spec: rocblas + lapack-rocm: + pkg_spec: rocsolver + lapack: + pkg_spec: cray-libsci + mpi-rocm-gtl: + pkg_spec: cray-mpich+gtl + mpi-rocm-no-gtl: + pkg_spec: cray-mpich~gtl + mpi-gcc: + pkg_spec: cray-mpich~gtl + fftw: + pkg_spec: intel-oneapi-mkl +"""