Run black and require black formatting in CI (mongodb#338)

evergreen.yml

@@ -151,6 +151,10 @@ tasks:
   commands:
   - func: f_lint_workloads
 
+- name: t_lint_python
+  commands:
+  - func: f_lint_python
+
 - name: t_cmake_test
   commands:
   - func: f_cmake_test
@@ -219,6 +223,7 @@ task_groups:
   # run after t_compile.
   - t_compile
   - t_python_test
+  - t_lint_python
   - t_lint_workloads
   - t_cmake_test
 
@@ -234,6 +239,7 @@ task_groups:
   - t_compile
   - t_mongo_server
   - t_python_test
+  - t_lint_python
   - t_cmake_test
   - t_integration_test_standalone
   - t_integration_test_single_node_replset
@@ -503,6 +509,31 @@ functions:
 
         lint-workloads
 
+  ##
+  # Lint python
+  ##
+  f_lint_python:
+  - command: shell.exec
+    params:
+      continue_on_err: true
+      working_dir: src
+      shell: bash
+      script: |
+        set -eo pipefail
+
+        export PATH=/opt/mongodbtoolchain/v3/bin:$PATH
+
+        python3 -m virtualenv venv
+        source venv/bin/activate
+
+        # Install Genny python scripts into the virtualenv
+        pushd src/python
+          python3 setup.py install
+        popd
+
+        black --check src/python src/lamplib
+
+
   ##
   # Reports test results to evergreen API.
   #

pyproject.toml

@@ -0,0 +1,3 @@
+[tool.black]
+line-length = 100
+target-version = ['py37']

src/lamplib/__main__.py

@@ -11,39 +11,49 @@
 from context import Context
 from parser import add_args_to_context
 
+
 def check_venv(args):
-    if not 'VIRTUAL_ENV' in os.environ and not args.run_global:
-        logging.error('Tried to execute without active virtualenv. If you want to run lamp '
-                      'without a virtualenv, use the --run-global option.')
+    if not "VIRTUAL_ENV" in os.environ and not args.run_global:
+        logging.error(
+            "Tried to execute without active virtualenv. If you want to run lamp "
+            "without a virtualenv, use the --run-global option."
+        )
         sys.exit(1)
 
+
 def run_self_test():
-    res = subprocess.run(['python3', '-m', 'unittest'],
-                         cwd=os.path.dirname(os.path.abspath(__file__)))
+    res = subprocess.run(
+        ["python3", "-m", "unittest"], cwd=os.path.dirname(os.path.abspath(__file__))
+    )
     res.check_returncode()
     sys.exit(0)
 
+
 def python_version_string():
-    return '.'.join(map(str, sys.version_info))[0:5]
+    return ".".join(map(str, sys.version_info))[0:5]
+
 
 def validate_environment():
     # Check Python version
     if not sys.version_info >= (3, 7):
-        raise OSError('Detected Python version {version} less than 3.7. Please delete '
-                      'the virtualenv and run lamp again.'.format(version=python_version_string()))
+        raise OSError(
+            "Detected Python version {version} less than 3.7. Please delete "
+            "the virtualenv and run lamp again.".format(version=python_version_string())
+        )
 
     # Check the macOS version. Non-mac platforms return a tuple of empty strings
     # for platform.mac_ver().
     if platform.mac_ver()[0]:
-        release_triplet = platform.mac_ver()[0].split('.')
+        release_triplet = platform.mac_ver()[0].split(".")
         if int(release_triplet[1]) < 14:
             # You could technically compile clang or gcc yourself on an older version
             # of macOS, but it's untested so we might as well just enforce
             # a blanket minimum macOS version for simplicity.
-            logging.error('Genny requires macOS 10.14 Mojave or newer')
+            logging.error("Genny requires macOS 10.14 Mojave or newer")
             sys.exit(1)
     return
 
+
 def main():
     validate_environment()
 
@@ -55,11 +65,11 @@ def main():
     # Pass around Context instead of using the global one to facilitate testing.
     context = Context
 
-    check_venv(args)        
+    check_venv(args)
 
     # Execute the minimum amount of code possible to run self tests to minimize
     # untestable code (i.e. code that runs the self-test).
-    if args.subcommand == 'self-test':
+    if args.subcommand == "self-test":
         run_self_test()
 
     toolchain_downloader = ToolchainDownloader(os_family, args.linux_distro)
@@ -73,29 +83,33 @@ def main():
         sys.exit(1)
     curator_path = curator_downloader.result_dir
 
-
     if not args.subcommand:
-        logging.info('No subcommand specified; running cmake, compile and install')
-        tasks.cmake(context, toolchain_dir=toolchain_dir,
-                    env=compile_env, cmdline_cmake_args=cmake_args)
+        logging.info("No subcommand specified; running cmake, compile and install")
+        tasks.cmake(
+            context, toolchain_dir=toolchain_dir, env=compile_env, cmdline_cmake_args=cmake_args
+        )
         tasks.compile_all(context, compile_env)
         tasks.install(context, compile_env)
-    elif args.subcommand == 'clean':
+    elif args.subcommand == "clean":
         tasks.clean(context, compile_env)
     else:
         tasks.compile_all(context, compile_env)
-        if args.subcommand == 'install':
+        if args.subcommand == "install":
             tasks.install(context, compile_env)
-        elif args.subcommand == 'cmake-test':
+        elif args.subcommand == "cmake-test":
             tasks.run_tests.cmake_test(compile_env)
-        elif args.subcommand == 'benchmark-test':
+        elif args.subcommand == "benchmark-test":
             tasks.run_tests.benchmark_test(compile_env)
-        elif args.subcommand == 'resmoke-test':
-            tasks.run_tests.resmoke_test(compile_env, suites=args.resmoke_suites,
-                                         mongo_dir=args.resmoke_mongo_dir, is_cnats=args.resmoke_cnats)
+        elif args.subcommand == "resmoke-test":
+            tasks.run_tests.resmoke_test(
+                compile_env,
+                suites=args.resmoke_suites,
+                mongo_dir=args.resmoke_mongo_dir,
+                is_cnats=args.resmoke_cnats,
+            )
         else:
-            raise ValueError('Unknown subcommand: ', args.subcommand)
+            raise ValueError("Unknown subcommand: ", args.subcommand)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

src/lamplib/context.py

@@ -2,38 +2,35 @@
 import os
 
 # Map of platform.system() to vcpkg's OS names.
-_triplet_os_map = {
-    'Darwin': 'osx',
-    'Linux': 'linux',
-    'NT': 'windows'
-}
+_triplet_os_map = {"Darwin": "osx", "Linux": "linux", "NT": "windows"}
 
 
 # Define complex operations as private methods on the module to keep the
 # public Context object clean.
 def _create_compile_environment(triplet_os, toolchain_dir):
     env = os.environ.copy()
-    paths = [env['PATH']]
+    paths = [env["PATH"]]
 
     # For mongodbtoolchain compiler (if there).
-    paths.insert(0, '/opt/mongodbtoolchain/v3/bin')
+    paths.insert(0, "/opt/mongodbtoolchain/v3/bin")
 
     # For cmake and ctest
     cmake_bin_relative_dir = {
-        'linux': 'downloads/tools/cmake-3.13.3-linux/cmake-3.13.3-Linux-x86_64/bin',
-        'osx': 'downloads/tools/cmake-3.13.3-osx/cmake-3.13.3-Darwin-x86_64/CMake.app/Contents/bin'
+        "linux": "downloads/tools/cmake-3.13.3-linux/cmake-3.13.3-Linux-x86_64/bin",
+        "osx": "downloads/tools/cmake-3.13.3-osx/cmake-3.13.3-Darwin-x86_64/CMake.app/Contents/bin",
     }[triplet_os]
     paths.insert(0, os.path.join(toolchain_dir, cmake_bin_relative_dir))
 
     # For ninja
-    ninja_bin_dir = os.path.join(toolchain_dir,
-                                 'downloads/tools/ninja-1.8.2-{}:'.format(triplet_os))
+    ninja_bin_dir = os.path.join(
+        toolchain_dir, "downloads/tools/ninja-1.8.2-{}:".format(triplet_os)
+    )
     paths.insert(0, ninja_bin_dir)
 
-    env['PATH'] = ':'.join(paths)
-    env['NINJA_STATUS'] = '[%f/%t (%p) %es] '  # make the ninja output even nicer
+    env["PATH"] = ":".join(paths)
+    env["NINJA_STATUS"] = "[%f/%t (%p) %es] "  # make the ninja output even nicer
 
-    logging.debug('Using environment: %s', env)
+    logging.debug("Using environment: %s", env)
     return env
 
 
@@ -53,10 +50,12 @@ def get_compile_environment(toolchain_dir=None):
         if not Context._compile_environment:
             if not toolchain_dir:
                 raise ValueError(
-                    'toolchain_dir must be specified when getting the compile environment for the '
-                    'first time')
-            Context._compile_environment = _create_compile_environment(Context.TRIPLET_OS,
-                                                                       toolchain_dir)
+                    "toolchain_dir must be specified when getting the compile environment for the "
+                    "first time"
+                )
+            Context._compile_environment = _create_compile_environment(
+                Context.TRIPLET_OS, toolchain_dir
+            )
         return Context._compile_environment
 
     # Helper methods

src/lamplib/parser.py

@@ -6,56 +6,77 @@
 
 def parse_args(args, os_family):
     parser = argparse.ArgumentParser(
-        description='Script for building genny',
-        epilog='Unknown positional arguments will be forwarded verbatim to the cmake'
-               ' invocation where relevant'
+        description="Script for building genny",
+        epilog="Unknown positional arguments will be forwarded verbatim to the cmake"
+        " invocation where relevant",
     )
 
     # Python can't natively check the distros of our supported platforms.
     # See https://bugs.python.org/issue18872 for more info.
-    parser.add_argument('-d', '--linux-distro',
-                        choices=['ubuntu1804', 'archlinux', 'rhel8', 'rhel70', 'rhel62',
-                                 'amazon2', 'not-linux'],
-                        help='specify the linux distro you\'re on; if your system isn\'t available,'
-                             ' please contact us at #workload-generation')
-    parser.add_argument('-v', '--verbose', action='store_true')
-    parser.add_argument('-g', '--run-global', action='store_true',
-                        help='allow installation outside of a virtualenv')
-    parser.add_argument('-i', '--ignore-toolchain-version', action='store_true',
-                        help='ignore the toolchain version, useful for testing toolchain changes')
-    parser.add_argument('-b', '--build-system',
-                        choices=['make', 'ninja'], default='ninja',
-                        help='Which build-system to use for compilation. May need to use make for '
-                             'IDEs.')
-    parser.add_argument('-s', '--sanitizer', choices=['asan', 'tsan', 'ubsan'])
+    parser.add_argument(
+        "-d",
+        "--linux-distro",
+        choices=["ubuntu1804", "archlinux", "rhel8", "rhel70", "rhel62", "amazon2", "not-linux"],
+        help="specify the linux distro you're on; if your system isn't available,"
+        " please contact us at #workload-generation",
+    )
+    parser.add_argument("-v", "--verbose", action="store_true")
+    parser.add_argument(
+        "-g", "--run-global", action="store_true", help="allow installation outside of a virtualenv"
+    )
+    parser.add_argument(
+        "-i",
+        "--ignore-toolchain-version",
+        action="store_true",
+        help="ignore the toolchain version, useful for testing toolchain changes",
+    )
+    parser.add_argument(
+        "-b",
+        "--build-system",
+        choices=["make", "ninja"],
+        default="ninja",
+        help="Which build-system to use for compilation. May need to use make for " "IDEs.",
+    )
+    parser.add_argument("-s", "--sanitizer", choices=["asan", "tsan", "ubsan"])
 
     subparsers = parser.add_subparsers(
-        dest='subcommand',
-        description='subcommands perform specific actions; make sure you run this script without '
-                    'any subcommand first to initialize the environment')
+        dest="subcommand",
+        description="subcommands perform specific actions; make sure you run this script without "
+        "any subcommand first to initialize the environment",
+    )
     subparsers.add_parser(
-        'cmake-test', help='run cmake unit tests that don\'t connect to a MongoDB cluster')
-    subparsers.add_parser('benchmark-test', help='run benchmark unit tests')
+        "cmake-test", help="run cmake unit tests that don't connect to a MongoDB cluster"
+    )
+    subparsers.add_parser("benchmark-test", help="run benchmark unit tests")
 
     resmoke_test_parser = subparsers.add_parser(
-        'resmoke-test', help='run cmake unit tests that connect to a MongoDB cluster')
+        "resmoke-test", help="run cmake unit tests that connect to a MongoDB cluster"
+    )
     group = resmoke_test_parser.add_mutually_exclusive_group()
-    group.add_argument('--suites', dest='resmoke_suites',
-                       help='equivalent to resmoke.py\'s "--suites" option')
-    group.add_argument('--create-new-actor-test-suite', action='store_true', dest='resmoke_cnats',
-                       help='Run the "genny_create_new_actor" resmoke test suite,'
-                            ' incompatible with the --suites options')
-    resmoke_test_parser.add_argument('--mongo-dir', dest='resmoke_mongo_dir',
-                       help='path to the mongo repo, which contains buildscripts/resmoke.py')
+    group.add_argument(
+        "--suites", dest="resmoke_suites", help='equivalent to resmoke.py\'s "--suites" option'
+    )
+    group.add_argument(
+        "--create-new-actor-test-suite",
+        action="store_true",
+        dest="resmoke_cnats",
+        help='Run the "genny_create_new_actor" resmoke test suite,'
+        " incompatible with the --suites options",
+    )
+    resmoke_test_parser.add_argument(
+        "--mongo-dir",
+        dest="resmoke_mongo_dir",
+        help="path to the mongo repo, which contains buildscripts/resmoke.py",
+    )
 
-    subparsers.add_parser('install', help='just run the install step for genny')
-    subparsers.add_parser('clean', help='cleanup existing build')
-    subparsers.add_parser('self-test', help='run lamplib unittests')
+    subparsers.add_parser("install", help="just run the install step for genny")
+    subparsers.add_parser("clean", help="cleanup existing build")
+    subparsers.add_parser("self-test", help="run lamplib unittests")
 
     known_args, unknown_args = parser.parse_known_args(args)
 
-    if os_family == 'Linux' and not known_args.subcommand and not known_args.linux_distro:
-        raise ValueError('--linux-distro must be specified on Linux')
+    if os_family == "Linux" and not known_args.subcommand and not known_args.linux_distro:
+        raise ValueError("--linux-distro must be specified on Linux")
 
     return known_args, unknown_args