Add command: benchpark system init (#298)

* Create Spec object with subclasses ExperimentSpec and SystemSpec

Spec objects have corrosponding ConcreteSpecObjects, which are immutable.
Spec objects use the same syntax as Spack specs for describing variants of systems
and experiments that can be instantiated to a directory

* Create Experiment and System objects

Experiment and System objects can be parametrized by variants
Experiment and System objects can be described by the corrosponding specs
These objects contain lifecycle methods that create the yaml files ingested by ramble
Experiment and System objects live in Repos from `ramble.repository.Repo`

* Bootstrap Ramble and Spack

Bootstrap download Ramble and Spack to ~/.benchpark directory to import
Ramble imports are executed normally
Spack imports have to be manually imported by path to avoid conflict with Ramble internals

* Refactor benchpark commands

Benchpark commands defined in `benchpark.main` from lib directory

* `benchpark-python` option

* Implement System and Experiment examples

Systems for Tioga and AWS with variants
Experiment for Saxpy with variants

* `benchpark system init` command

Takes a SystemSpec as input
Concretizes to a ConcreteSystemSpec
Generates a System associated with the ConcreteSystemSpec
Creates all yaml files associated with that System
These yaml files can be ingested as a system to `benchpark setup`

---------

Co-authored-by: Gregory Becker <[email protected]>
Co-authored-by: Alec Scott <[email protected]>

bin/benchpark-python

@@ -0,0 +1,21 @@
+#!/bin/sh
+#
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+#
+# benchpark-python
+#
+# If you want to write your own executable Python script that uses Benchpark
+# modules, on Mac OS or maybe some others, you may be able to do it like
+# this:
+#
+#   #!/usr/bin/env benchpark-python
+#
+# This is compatible across platforms.
+#
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+export PYTHONPATH="${SCRIPT_DIR}/../lib":$PYTHONPATH
+exec python3 -i "$@"

lib/benchpark/cmd/system.py

@@ -0,0 +1,68 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import shutil
+import sys
+
+import benchpark.system
+import benchpark.spec
+
+
+def system_init(args):
+    system_spec = benchpark.spec.SystemSpec(" ".join(args.spec))
+    system_spec = system_spec.concretize()
+
+    system = system_spec.system
+    system.initialize()
+
+    if args.basedir:
+        base = args.basedir
+        sysdir = system.system_id()
+        destdir = os.path.join(base, sysdir)
+    elif args.dest:
+        destdir = args.dest
+    else:
+        raise ValueError("Must specify one of: --dest, --basedir")
+
+    try:
+        os.mkdir(destdir)
+        system.generate_description(destdir)
+    except FileExistsError:
+        print(f"Abort: system description dir already exists ({destdir})")
+        sys.exit(1)
+    except Exception:
+        # If there was a failure, remove any partially-generated resources
+        shutil.rmtree(destdir)
+        raise
+
+
+def system_list(args):
+    raise NotImplementedError("'benchpark system list' is not available")
+
+
+def setup_parser(root_parser):
+    system_subparser = root_parser.add_subparsers(dest="system_subcommand")
+
+    init_parser = system_subparser.add_parser("init")
+    init_parser.add_argument("--dest", help="Place all system files here directly")
+    init_parser.add_argument(
+        "--basedir", help="Generate a system dir under this, and place all files there"
+    )
+
+    init_parser.add_argument("spec", nargs="+", help="System spec")
+
+    system_subparser.add_parser("list")
+
+
+def command(args):
+    actions = {
+        "init": system_init,
+        "list": system_list,
+    }
+    if args.system_subcommand in actions:
+        actions[args.system_subcommand](args)
+    else:
+        raise ValueError(f"Unknown subcommand for 'system': {args.system_subcommand}")

lib/benchpark/directives.py

@@ -0,0 +1,202 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import collections.abc
+import inspect
+import os
+import re
+from typing import Any, Callable, Optional, Tuple, Union
+
+import benchpark.spec
+import benchpark.paths
+import benchpark.repo
+import benchpark.runtime
+import benchpark.variant
+
+import ramble.language.language_base
+import ramble.language.language_helpers
+import ramble.language.shared_language
+from ramble.language.language_base import DirectiveError
+
+
+# TODO remove this when it is added to ramble.lang (when ramble updates from spack)
+class classproperty:
+    """Non-data descriptor to evaluate a class-level property. The function that performs
+    the evaluation is injected at creation time and take an instance (could be None) and
+    an owner (i.e. the class that originated the instance)
+    """
+
+    def __init__(self, callback):
+        self.callback = callback
+
+    def __get__(self, instance, owner):
+        return self.callback(owner)
+
+
+class DirectiveMeta(ramble.language.shared_language.SharedMeta):
+    """
+    metaclass for supporting directives (e.g., depends_on) and phases
+    """
+
+    _directive_names = set()
+    _directives_to_be_executed = []
+
+    # Hack to be able to use SharedMeta outside of Ramble
+    # will ask Ramble to implement fix on their end and then we can remove this
+    def __init__(self, *args, **kwargs):
+        with benchpark.repo.override_ramble_hardcoded_globals():
+            super(DirectiveMeta, self).__init__(*args, **kwargs)
+
+
+benchpark_directive = DirectiveMeta.directive
+
+
+@benchpark_directive("variants")
+def variant(
+    name: str,
+    default: Optional[Any] = None,
+    description: str = "",
+    values: Optional[Union[collections.abc.Sequence, Callable[[Any], bool]]] = None,
+    multi: Optional[bool] = None,
+    validator: Optional[Callable[[str, str, Tuple[Any, ...]], None]] = None,
+    when: Optional[Union[str, bool]] = None,
+    sticky: bool = False,
+):
+    """Define a variant.
+    Can specify a default value as well as a text description.
+    Args:
+        name: Name of the variant
+        default: Default value for the variant, if not specified otherwise the default will be
+            False for a boolean variant and 'nothing' for a multi-valued variant
+        description: Description of the purpose of the variant
+        values: Either a tuple of strings containing the allowed values, or a callable accepting
+            one value and returning True if it is valid
+        multi: If False only one value per spec is allowed for this variant
+        validator: Optional group validator to enforce additional logic. It receives the experiment
+            name, the variant name and a tuple of values and should raise an instance of BenchparkError
+            if the group doesn't meet the additional constraints
+        when: Optional condition on which the variant applies
+        sticky: The variant should not be changed by the concretizer to find a valid concrete spec
+    Raises:
+        DirectiveError: If arguments passed to the directive are invalid
+    """
+
+    def format_error(msg, pkg):
+        msg += " @*r{{[{0}, variant '{1}']}}"
+        return msg.format(pkg.name, name)
+
+    def _always_true(_x):
+        return True
+
+    # Ensure we have a sequence of allowed variant values, or a
+    # predicate for it.
+    if values is None:
+        if str(default).upper() in ("TRUE", "FALSE"):
+            values = (True, False)
+        else:
+            values = _always_true
+
+    # The object defining variant values might supply its own defaults for
+    # all the other arguments. Ensure we have no conflicting definitions
+    # in place.
+    for argument in ("default", "multi", "validator"):
+        # TODO: we can consider treating 'default' differently from other
+        # TODO: attributes and let a packager decide whether to use the fluent
+        # TODO: interface or the directive argument
+        if hasattr(values, argument) and locals()[argument] is not None:
+
+            def _raise_argument_error(pkg):
+                msg = (
+                    "Remove specification of {0} argument: it is handled "
+                    "by an attribute of the 'values' argument"
+                )
+                raise DirectiveError(format_error(msg.format(argument), pkg))
+
+            return _raise_argument_error
+
+    # Allow for the object defining the allowed values to supply its own
+    # default value and group validator, say if it supports multiple values.
+    default = getattr(values, "default", default)
+    validator = getattr(values, "validator", validator)
+    multi = getattr(values, "multi", bool(multi))
+
+    # Here we sanitize against a default value being either None
+    # or the empty string, as the former indicates that a default
+    # was not set while the latter will make the variant unparsable
+    # from the command line
+    if default is None or default == "":
+
+        def _raise_default_not_set(pkg):
+            if default is None:
+                msg = "either a default was not explicitly set, or 'None' was used"
+            elif default == "":
+                msg = "the default cannot be an empty string"
+            raise DirectiveError(format_error(msg, pkg))
+
+        return _raise_default_not_set
+
+    description = str(description).strip()
+
+    def _execute_variant(pkg):
+        if not re.match(benchpark.spec.IDENTIFIER, name):
+            directive = "variant"
+            msg = "Invalid variant name in {0}: '{1}'"
+            raise DirectiveError(directive, msg.format(pkg.name, name))
+
+        pkg.variants[name] = benchpark.variant.Variant(
+            name, default, description, values, multi, validator, sticky
+        )
+
+    return _execute_variant
+
+
+class ExperimentSystemBase(metaclass=DirectiveMeta):
+    @classproperty
+    def template_dir(cls):
+        """Directory where the experiment/system.py file lives."""
+        return os.path.abspath(os.path.dirname(cls.module.__file__))
+
+    @classproperty
+    def module(cls):
+        """Module object (not just the name) that this Experiment/System is
+        defined in.
+        """
+        return __import__(cls.__module__, fromlist=[cls.__name__])
+
+    @classproperty
+    def namespace(cls):
+        """namespace for the Experiment/System, which identifies its repo."""
+        parts = cls.__module__.split(".")
+        return ".".join(parts[2:-1])
+
+    @classproperty
+    def fullname(cls):
+        """Name of this Experiment/System, including the namespace"""
+        return f"{cls.namespace}.{cls.name}"
+
+    @classproperty
+    def fullnames(cls):
+        """Fullnames for this Experiment/System and any from which it inherits."""
+        fullnames = []
+        for cls in inspect.getmro(cls):
+            namespace = getattr(cls, "namespace", None)
+            if namespace:
+                fullnames.append(f"{namespace}.{cls.name}")
+            if namespace == "builtin":
+                # builtin packages cannot inherit from other repos
+                break
+        return fullnames
+
+    @classproperty
+    def name(cls):
+        """The name of this Experiment/System.
+        This is the name of its Python module, without the containing module
+        names.
+        """
+        if cls._name is None:
+            cls._name = cls.module.__name__
+            if "." in cls._name:
+                cls._name = cls._name[cls._name.rindex(".") + 1 :]
+        return cls._name

lib/benchpark/error.py

@@ -0,0 +1,14 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+#: at what level we should write stack traces or short error messages
+#: this is module-scoped because it needs to be set very early
+debug = 0
+
+
+class BenchparkError(Exception):
+    """This is the superclass for all Benchpark errors.
+    Subclasses can be found in the modules they have to do with.
+    """