TIG-2835 Enforce Documentation for Genny workloads (mongodb#539)

.yamllint

@@ -1,6 +1,8 @@
 extends: default
 
 rules:
+  braces:
+    max-spaces-inside: 1
   colons:
     max-spaces-before: 0
     max-spaces-after: 1

evergreen.yml

@@ -230,6 +230,7 @@ task_groups:
   - t_compile
   - t_python_test
   - t_lint_python
+  - t_lint_workloads
   - t_cmake_test
   - t_integration_test_single_node_replset
   - t_integration_test_three_node_replset
@@ -339,7 +340,6 @@ functions:
       binary: ./src/genny/run-genny
       args:
       - -v
-      # TODO: this fails right now - should show up in patch-build; need to remove this task and TODO as a ticket
       - lint-yaml
 
   ##

run-genny

@@ -26,7 +26,7 @@ _print_diagnostics() {
     if [ -n "$_HAVE_PRINTED_DIAGNOSTICS" ]; then
         return
     fi
-    echo >&2 "If you're stuck, please reach out to the #workload-generation MongoDB slack channel and paste this output."
+    echo >&2 "If you're stuck, please reach out to the #performance-tooling-users MongoDB slack channel and paste this output."
     echo >&2 ""
     echo >&2 "  git rev-parse HEAD: $(git rev-parse HEAD)"
     echo >&2 "  uname -a:           $(uname -a)"

src/lamplib/src/genny/tasks/create-new-actor.sh

@@ -461,7 +461,10 @@ create_workload_yml() {
     actor_name="$1"
 cat << EOF > "$GENNY_REPO_ROOT/src/workloads/docs/${actor_name}.yml"
 SchemaVersion: 2018-07-01
-Owner: TODO put your github team name here e.g. @mongodb/stm
+Owner: TODO put your github team name here e.g. @10gen/dev-prod-tips
+Description: |
+  TODO describe your workload. For an example description, check out
+  src/workloads/selftests/GennyOverhead.yml.
 
 # TODO: delete this file or add a meaningful workload using or
 #       demonstrating your Actor
@@ -615,19 +618,17 @@ cat << EOF
 
 Build and test ${actor_name} with the following command:
 
-    # TODO: these paths aren't right any more
-    ./scripts/lamp
-    ./scripts/lamp cmake-test
-    ./scripts/get-mongo-source.sh
-    ./scripts/lamp resmoke-test --suites src/resmokeconfig/genny_standalone.yml
+    ./run-genny install
+    ./run-genny cmake-test
+    ./run-genny resmoke-test --suites src/resmokeconfig/genny_standalone.yml
 
 The resmoke-test will fail because there is a "hidden" bug in the generated
 integration-test-case that you are expected to find as a part of reading through
 the generated code.
 
 Run your workload as follows:
 
-    ./dist/bin/genny run \\
+    ./run-genny workload -- run \\
         --workload-file       ./src/workloads/docs/${actor_name}.yml \\
         --metrics-format      csv \\
         --metrics-output-file build/genny-metrics.csv \\

src/lamplib/src/genny/tasks/yaml_linter.py

@@ -1,6 +1,7 @@
 import os
 import os.path as path
 import sys
+import yaml
 
 import structlog
 import yamllint.cli
@@ -9,42 +10,81 @@
 
 
 def main(genny_repo_root: str):
-    yaml_dirs = [
+    workload_dirs = [
         path.join(genny_repo_root, "src", "workloads"),
         path.join(genny_repo_root, "src", "phases"),
-        path.join(genny_repo_root, "src", "resmokeconfig"),
     ]
+    resmoke_dirs = [path.join(genny_repo_root, "src", "resmokeconfig")]
+    evergreen_dirs = [path.join(genny_repo_root, "evergreen.yml")]
 
-    yaml_files = [path.join(os.getcwd(), "evergreen.yml")]
+    workload_yamls, workload_error = _traverse_yamls(workload_dirs)
+    resmoke_yamls, resmoke_error = _traverse_yamls(resmoke_dirs)
+    evergreen_yamls, evergreen_error = _traverse_yamls(evergreen_dirs)
+    all_yamls = workload_yamls + resmoke_yamls + evergreen_yamls
 
-    has_error = False
-
-    for yaml_dir in yaml_dirs:
-        for dirpath, dirnames, filenames in os.walk(yaml_dir):
-            for filename in filenames:
-                if filename.endswith(".yaml"):
-                    SLOG.error("All YAML files should have the .yml extension", found=filename)
-                    # Don't error immediately so all violations can be printed with one run
-                    # of this script.
-                    has_error = True
-                elif filename.endswith(".yml"):
-                    yaml_files.append(path.join(dirpath, filename))
-
-    if has_error:
+    if workload_error or resmoke_error or evergreen_error:
+        SLOG.error(
+            "Found invalidly-named yaml files. Please correct and rerun ./run-genny lint-yaml."
+        )
         sys.exit(1)
 
-    if len(yaml_files) == 0:
-        SLOG.error("Did not find any YAML files to lint", in_dirs=yaml_dirs)
-        raise Exception("No yamls found")
-
-    config_file_path = path.join(os.getcwd(), ".yamllint")
+    all_have_descriptions = True
+    for workload_yaml in workload_yamls:
+        if not check_description(workload_yaml):
+            all_have_descriptions = False
+    if not all_have_descriptions:
+        SLOG.error(
+            "The above YAML workloads lack a Description field. This field should be populated with a human-readable description "
+            "of the workload and its output metrics. After doing so, please re-run ./run-genny lint-yaml"
+        )
+        sys.exit(1)
 
-    yamllint_argv = ["--strict", "--config-file", config_file_path] + yaml_files
+    config_file_path = path.join(genny_repo_root, ".yamllint")
+    yamllint_argv = ["--strict", "--config-file", config_file_path] + all_yamls
 
     SLOG.info(
         "Linting workload YAML files with yamllint",
-        count=len(yaml_files),
+        count=len(all_yamls),
         yamllint_argv=yamllint_argv,
     )
 
     yamllint.cli.run(yamllint_argv)
+
+
+def check_description(yaml_path):
+    workload = _load_yaml(yaml_path)
+    if "Description" not in workload:
+        SLOG.error(f"Genny workload {yaml_path} lacks a Description field.")
+        return False
+    return True
+
+
+def _traverse_yamls(roots):
+    def check_filename(filename):
+        if filename.endswith(".yaml"):
+            SLOG.error("All YAML files should have the .yml extension", found=filename)
+            return True
+        return False
+
+    yaml_files = []
+    has_error = False
+    for root in roots:
+        if os.path.isfile(root):
+            return [root], check_filename(root)
+        for dirpath, dirnames, filenames in os.walk(root):
+            for filename in filenames:
+                # Don't error immediately so all violations can be printed with one run
+                # of this script.
+                has_error = check_filename(filename)
+                if filename.endswith(".yml"):
+                    yaml_files.append(path.join(dirpath, filename))
+    if len(yaml_files) == 0:
+        SLOG.error("Did not find any YAML files to lint", in_dirs=directories)
+        raise Exception("No yamls found")
+    return yaml_files, has_error
+
+
+def _load_yaml(yaml_path):
+    with open(yaml_path) as file:
+        workload = yaml.safe_load(file)
+        return workload

src/phases/HelloWorld/ExamplePhase2.yml

@@ -1,5 +1,6 @@
-# Example to illustrate how PhaseConfig composition works.
 SchemaVersion: 2018-07-01
+Description: |
+  Example phase to illustrate how PhaseConfig composition works.
 
 UseMe:
   Message: Hello Phase 2

src/phases/execution/CreateIndexPhase.yml

@@ -2,6 +2,8 @@
 
 SchemaVersion: 2018-07-01
 Owner: "@mongodb/server-execution"
+Description: |
+  TODO: TIG-3322
 
 InsertData:
   Repeat: 1

src/phases/scale/DesignDocWorkloadPhases.yml

@@ -1,4 +1,6 @@
 SchemaVersion: 2018-07-01
+Description: |
+  TODO: TIG-3318
 
 Document: &Doc  # Documents are approximately 1 KB in size
   t: {^RandomInt: {min: 0, max: 10}}

src/phases/scale/LargeScalePhases.yml

@@ -1,5 +1,7 @@
 SchemaVersion: 2018-07-01
 Owner: Storage Engines
+Description: |
+  TODO: TIG-3319
 
 # This is the set of shared phases for the Large Scale Workload Automation project.
 #
@@ -9,7 +11,7 @@ Owner: Storage Engines
 # This section contains definitions that are used in the following phases.
 GlobalDefaults:
   Random10KInt: &rand_10k_int {^RandomInt: {min: 0, max: 10000}}
-  Random10KInt: &rand_1k_int {^RandomInt: {min: 0, max: 1000}}
+  Random1KInt: &rand_1k_int {^RandomInt: {min: 0, max: 1000}}
   Random4ByteInt: &rand_4b_int {^RandomInt: {min: 0, max: 2147483647}}
   Random30String: &rand_30b_string {^RandomString: {length: 30}}
   Random130String: &rand_100b_string {^RandomString: {length: 100}}
@@ -28,15 +30,15 @@ GlobalDefaults:
     h: *rand_4b_int
     i: *rand_4b_int
   RollingIndexes: &RollingIndexes
-    - keys: {a: 1}
-    - keys: {b: 1}
-    - keys: {c: 1}
-    - keys: {d: 1}
-    - keys: {e: 1}
-    - keys: {f: 1}
-    - keys: {i: 1, g: 1}
-    - keys: {g: 1, h: 1}
-    - keys: {h: 1, i: 1}
+  - keys: {a: 1}
+  - keys: {b: 1}
+  - keys: {c: 1}
+  - keys: {d: 1}
+  - keys: {e: 1}
+  - keys: {f: 1}
+  - keys: {i: 1, g: 1}
+  - keys: {g: 1, h: 1}
+  - keys: {h: 1, i: 1}
 
   # Commonly used parameters.
   DatabaseParam: &DatabaseParam {^Parameter: {Name: "Database", Default: ""}}
@@ -153,7 +155,7 @@ HotDocumentUpdaterCmd:
     OperationCommand:
       WriteOperations:
       - WriteCommand: updateOne
-        Filter: {first : first}
+        Filter: {first: first}
         Update: {$set: {second: *rand_1k_int}}
   GlobalRate: *WritesParam
 
@@ -176,7 +178,7 @@ HotCollectionUpdaterCmd:
     OperationCommand:
       WriteOperations:
       - WriteCommand: insertOne
-        Document: { a : *rand_1k_int}
+        Document: { a: *rand_1k_int }
   GlobalRate: *WritesParam
 
 HotCollectionDeleterCmd:
@@ -249,8 +251,8 @@ SnapshotScannerAllCmd:
   Duration: *Duration
   Documents: 100000000000
   ScanType: snapshot
-  #GlobalRate: 1 per 24 hours   # CHANGE
-  #Note: using "2 per 1" hour kicks off two scans at the top of the hour!
+  # GlobalRate: 1 per 24 hours   # CHANGE
+  # Note: using "2 per 1" hour kicks off two scans at the top of the hour!
   GlobalRate: 1 per 30 minutes
 
 OplogTrailerCmd:

src/phases/scale/MixPhases.yml

@@ -1,4 +1,7 @@
 SchemaVersion: 2018-07-01
+Owner: "@mongodb/product-perf"
+Description: |
+  TODO: TIG-3318
 
 dbname: &dbname mix
 runtime: &runtime 7 minutes