implement clean output options

dev.py

@@ -20,7 +20,10 @@
 from devscripts.commands import add_subcommands
 
 parser = argparse.ArgumentParser(description="Helper-script for various tasks during development.")
-parser.add_argument("-q", "--quiet", default=False, action="store_true", help="Do not display commands.")
+parser.add_argument(
+    "--show-commands", default=False, action="store_true", help="Show commands being executed."
+)
+parser.add_argument("--show-output", default=False, action="store_true", help="Show ouptut of commands.")
 add_subcommands(parser, "devscripts.commands")
 args = parser.parse_args()
 

devscripts/commands/__init__.py

@@ -60,6 +60,7 @@ class DevCommand:
 
     @property
     def parser_parents(self) -> Sequence[argparse.ArgumentParser]:
+        """Argument parser parents, can be overwritten by subclasses."""
         return []
 
     def __init__(self, **kwargs: Any) -> None:
@@ -79,8 +80,10 @@ def exec(self, parser: argparse.ArgumentParser, args: argparse.Namespace) -> Non
             mod = importlib.import_module(mod_name)
             setattr(self, mod_name, mod)
 
-        if args.quiet:
-            config.OUTPUT_COMMANDS = False
+        if args.show_commands:
+            config.SHOW_COMMANDS = True
+        if args.show_output:
+            config.SHOW_COMMAND_OUTPUT = True
 
         try:
             self.handle(args)
@@ -176,6 +179,7 @@ def handle(self, args: argparse.Namespace) -> None:
 
 
 def add_subcommands(parser: argparse.ArgumentParser, path: str, dest: str = "command", **kwargs: Any) -> None:
+    """Function to add subcommands gin `path` to `parser`."""
     commands = parser.add_subparsers(dest=dest)
 
     # Get a list of submodules:

devscripts/commands/code_quality.py

@@ -36,15 +36,24 @@ def manage(self, *args: str) -> None:
         #   https://docs.djangoproject.com/en/4.0/releases/4.0/#miscellaneous
         python += ["-W", "ignore:The USE_L10N setting is deprecated."]  # pragma: only django<4.0
 
+        # Django 4.2 introduced a new way of handling storages
+        python += [
+            "-W",
+            "ignore:django.core.files.storage.get_storage_class is deprecated",
+        ]  # pragma: only django<4.2
+
         # kombu==5.2.4 uses the deprecated select interface. Should be fixed in the next release:
         #   https://github.com/celery/kombu/pull/1601
-        python += ["-W", "ignore:SelectableGroups dict interface is deprecated. Use select."]
+        # python += ["-W", "ignore:SelectableGroups dict interface is deprecated. Use select."]
 
         python.append(config.MANAGE_PY.relative_to(config.ROOT_DIR))
         python += args
         return self.run(*python)
 
     def handle(self, args: argparse.Namespace) -> None:
+        config.SHOW_COMMANDS = True
+        config.SHOW_COMMAND_OUTPUT = True
+
         self.run("isort", "--check-only", "--diff", ".")
         self.run("flake8", ".")
         self.run("black", "--check", ".")

devscripts/commands/init_demo.py

@@ -208,7 +208,6 @@ def handle(self, args: argparse.Namespace) -> None:
             [
                 "python",
                 "dev.py",
-                "--quiet",
                 "recreate-fixtures",
                 "--no-delay",
                 "--no-ocsp",

devscripts/config.py

@@ -30,7 +30,8 @@
 DEVSCRIPTS_DIR = ROOT_DIR / "devscripts"
 DEVSCRIPTS_FILES = DEVSCRIPTS_DIR / "files"
 
-OUTPUT_COMMANDS = True
+SHOW_COMMANDS = False
+SHOW_COMMAND_OUTPUT = False
 
 
 def minor_to_major(version: str) -> str:

devscripts/utils.py

@@ -80,7 +80,7 @@ def _wait_for(
         wait_for_cmd = shlex.split(jinja_env.from_string(command["command"]).render(**context))
 
         for i in range(0, 15):
-            wait_for_proc = run(wait_for_cmd, check=False, capture_output=True, **kwargs)
+            wait_for_proc = run(wait_for_cmd, check=False, **kwargs)
             if wait_for_proc.returncode == 0:
                 break
             time.sleep(1)
@@ -132,7 +132,7 @@ def console_include(path: str, context: Dict[str, Any]) -> Iterator[None]:
             # If a "wait_for" command is defined, don't run actual command until it succeeds
             _wait_for(command.get("wait_for"), env, context, env=shell_env)
 
-            run(args, capture_output=command.get("capture_output", True), input=stdin, env=shell_env)
+            run(args, input=stdin, env=shell_env)
 
             for cmd in command.get("after_command", []):
                 run(shlex.split(env.from_string(cmd).render(**context)))
@@ -142,7 +142,7 @@ def console_include(path: str, context: Dict[str, Any]) -> Iterator[None]:
         yield
     finally:
         for args in reversed(clean_commands):
-            run(args, check=False, capture_output=True)
+            run(args, check=False)
 
 
 def get_previous_release(current_release: Optional[str] = None) -> str:
@@ -193,8 +193,11 @@ def tmpdir() -> Iterator[str]:
 def run(args: Sequence[str], **kwargs: Any) -> "subprocess.CompletedProcess[Any]":
     """Shortcut for subprocess.run()."""
     kwargs.setdefault("check", True)
-    if config.OUTPUT_COMMANDS:
+    if config.SHOW_COMMANDS:
         print("+", shlex.join(args))
+    if not config.SHOW_COMMAND_OUTPUT and not kwargs.get("capture_output"):
+        kwargs.setdefault("stdout", subprocess.DEVNULL)
+        kwargs.setdefault("stderr", subprocess.DEVNULL)
     return subprocess.run(args, **kwargs)  # pylint: disable=subprocess-run-check
 
 

devscripts/validation/docker.py

@@ -104,13 +104,13 @@ def docker_cp(src: str, container: str, dest: str) -> None:
 def build_docker_image(release: str, prune: bool = True, build: bool = True) -> str:
     """Build the docker image."""
     if prune:
-        utils.run(["docker", "system", "prune", "-af"], capture_output=True)
+        utils.run(["docker", "system", "prune", "-af"])
 
     tag = f"{config.DOCKER_TAG}:{release}"
     if build:
         info("Building docker image...")
-        utils.run(["docker", "build", "-t", tag, "."], env={"DOCKER_BUILDKIT": "1"}, capture_output=True)
-        ok("Docker image built.")
+        utils.run(["docker", "build", "-t", tag, "."], env={"DOCKER_BUILDKIT": "1"})
+        ok(f"Docker image built as {tag}.")
     return tag
 
 
@@ -149,7 +149,6 @@ def validate_docker_image(release: str, docker_tag: str) -> int:
         with tut.run("start-dependencies.yaml"), tut.run("start-django-ca.yaml"), tut.run(
             "start-nginx.yaml"
         ), tut.run("setup-cas.yaml"):
-            input(os.getcwd())
             errors += _test_connectivity(standalone_src)
 
             print("Now running running django-ca, please visit:\n\n\thttp://localhost/admin\n")
@@ -159,13 +158,16 @@ def validate_docker_image(release: str, docker_tag: str) -> int:
 
 
 class Command(DevCommand):
+    """Class implementing the ``dev.py validate docker`` command."""
+
     modules = (("django_ca", "django-ca"),)
     django_ca: ModuleType
     help_text = "Validate Docker setup."
 
     @property
     def parser_parents(self) -> Sequence[argparse.ArgumentParser]:
-        return [self.parent.docker_options]  # type: ignore[attr-defined]  # set in the constructor
+        # pylint: disable-next=no-member  # set in the constructor of parent class
+        return [self.parent.docker_options]  # type: ignore[attr-defined]  # see pylint above
 
     def handle(self, args: argparse.Namespace) -> None:
         release = self.django_ca.__version__

devscripts/validation/docker_compose.py

@@ -41,7 +41,7 @@
 @contextmanager
 def _compose_up(remove_volumes: bool = True, **kwargs: Any) -> Iterator[None]:
     try:
-        utils.run(["docker", "compose", "up", "-d"], capture_output=True, **kwargs)
+        utils.run(["docker", "compose", "up", "-d"], **kwargs)
         yield
     finally:
         down = ["docker", "compose", "down"]
@@ -51,7 +51,7 @@ def _compose_up(remove_volumes: bool = True, **kwargs: Any) -> Iterator[None]:
         if "env" in kwargs:
             down_kwargs["env"] = kwargs["env"]
 
-        utils.run(down, capture_output=True, **down_kwargs)
+        utils.run(down, **down_kwargs)
 
 
 def _compose_exec(*args: str, **kwargs: Any) -> "subprocess.CompletedProcess[Any]":
@@ -63,18 +63,17 @@ def _manage(container: str, *args: str, **kwargs: Any) -> "subprocess.CompletedP
     return _compose_exec(container, "manage", *args, **kwargs)
 
 
-def _sign_cert(container: str, ca: str, csr: str) -> str:
+def _sign_cert(container: str, ca: str, csr: str, **kwargs: Any) -> str:
     subject = f"signed-in-{container}.{ca.lower()}.example.com"
 
     _manage(
         container,
         "sign_cert",
         f"--ca={ca}",
         f"--subject=/CN={subject}",
-        capture_output=True,
-        input=csr,
-        text=True,
+        input=csr.encode("utf-8"),
         compose_args=["-T"],
+        **kwargs,
     )
     return subject
 
@@ -84,15 +83,15 @@ def _run_py(container: str, code: str, env: Optional[Dict[str, str]] = None) ->
     return typing.cast(str, proc.stdout)  # is a str because of text=True above
 
 
-def _openssl_verify(ca_file: str, cert_file: str) -> "subprocess.CompletedProcess[Any]":
+def _openssl_verify(ca_file: str, cert_file: str, **kwargs: Any) -> "subprocess.CompletedProcess[Any]":
     return utils.run(
-        ["openssl", "verify", "-CAfile", ca_file, "-crl_download", "-crl_check", cert_file],
-        capture_output=True,
-        text=True,
+        ["openssl", "verify", "-CAfile", ca_file, "-crl_download", "-crl_check", cert_file], **kwargs
     )
 
 
-def _openssl_ocsp(ca_file: str, cert_file: str, url: str) -> "subprocess.CompletedProcess[Any]":
+def _openssl_ocsp(
+    ca_file: str, cert_file: str, url: str, **kwargs: Any
+) -> "subprocess.CompletedProcess[Any]":
     return utils.run(
         [
             "openssl",
@@ -107,8 +106,7 @@ def _openssl_ocsp(ca_file: str, cert_file: str, url: str) -> "subprocess.Complet
             "-url",
             url,
         ],
-        capture_output=True,
-        text=True,
+        **kwargs,
     )
 
 
@@ -123,8 +121,7 @@ def _validate_container_versions(release: str, env: Optional[Dict[str, str]] = N
         errors += err(f"backend container identifies as {backend_ver} instead of {release}.")
     if frontend_ver != release:
         errors += err(f"frontend container identifies as {frontend_ver} instead of {release}.")
-    if not errors:
-        ok(f"Container version: {backend_ver}")
+
     return errors
 
 
@@ -168,12 +165,12 @@ def _validate_crl_ocsp(ca_file: str, cert_file: str, cert_subject: str) -> None:
     time.sleep(1)  # give celery task some time
 
     # "openssl ocsp" always returns 0 if it retrieves a valid OCSP response, even if the cert is revoked
-    proc = _openssl_ocsp(ca_file, cert_file, ocsp_url)
+    proc = _openssl_ocsp(ca_file, cert_file, ocsp_url, capture_output=True, text=True)
     assert "Cert Status: revoked" in proc.stdout
 
     # Make sure that CRL validation fails now too
     try:
-        _openssl_verify(ca_file, cert_file)
+        _openssl_verify(ca_file, cert_file, capture_output=True, text=True)
     except subprocess.CalledProcessError as ex:
         # OpenSSL in Ubuntu 20.04 outputs this on stdout, in 22.04 it goes to stderr
         assert "verification failed" in ex.stdout or "verification failed" in ex.stderr
@@ -211,9 +208,9 @@ def _sign_certificates(csr: str) -> str:
     _sign_cert("frontend", "Intermediate", csr)
 
     try:
-        _sign_cert("frontend", "Root", csr)
+        _sign_cert("frontend", "Root", csr, capture_output=True)
     except subprocess.CalledProcessError as ex:
-        assert re.search(r"Root:.*Private key does not exist\.", ex.stderr)
+        assert re.search(rb"Root:.*Private key does not exist\.", ex.stderr)
     else:
         raise RuntimeError("Was able to sign root cert in frontend.")
 
@@ -276,8 +273,8 @@ def test_tutorial(release: str) -> int:  # pylint: disable=too-many-statements,t
             ok("Containers seem to have started properly.")
 
             # Check that we didn't forget any migrations
-            _manage("backend", "makemigrations", "--check", capture_output=True)
-            _manage("frontend", "makemigrations", "--check", capture_output=True)
+            _manage("backend", "makemigrations", "--check")
+            _manage("frontend", "makemigrations", "--check")
 
             # Validate that the container versions match the expected version
             errors += _validate_container_versions(release)
@@ -323,10 +320,8 @@ def test_tutorial(release: str) -> int:  # pylint: disable=too-many-statements,t
                 # Test CRL and OCSP validation
                 _validate_crl_ocsp("root.pem", f"{cert_subject}.pem", cert_subject)
 
-                utils.run(["docker", "compose", "down"], capture_output=True)
-                utils.run(
-                    ["docker", "compose", "up", "-d"], env={"DJANGO_CA_VERSION": release}, capture_output=True
-                )
+                utils.run(["docker", "compose", "down"])
+                utils.run(["docker", "compose", "up", "-d"], env={"DJANGO_CA_VERSION": release})
                 ok("Restarted docker containers.")
 
                 # Finally some manual testing
@@ -385,8 +380,10 @@ def test_update(release: str) -> int:
             # Start previous version
             with _compose_up(remove_volumes=False, env=dict(os.environ, DJANGO_CA_VERSION=last_release)):
                 # Make sure we have started the right version
+                info(f"Start previous version ({last_release}).")
                 _validate_container_versions(last_release)
 
+                info("Create test data...")
                 docker_cp(str(standalone_dir / "create-testdata.py"), backend, standalone_dest)
                 docker_cp(str(standalone_dir / "create-testdata.py"), frontend, standalone_dest)
                 _compose_exec("backend", "./create-testdata.py", "--env", "backend")
@@ -396,9 +393,11 @@ def test_update(release: str) -> int:
             shutil.copy(config.ROOT_DIR / "docker-compose.yml", tmpdir)
 
             with _compose_up(env=dict(os.environ, DJANGO_CA_VERSION=release)):
+                info(f"Start current version ({last_release}).")
                 # Make sure we have the new version
                 _validate_container_versions(release)
 
+                info("Validate test data...")
                 docker_cp(str(standalone_dir / "validate-testdata.py"), backend, standalone_dest)
                 docker_cp(str(standalone_dir / "validate-testdata.py"), frontend, standalone_dest)
                 _compose_exec("backend", "./validate-testdata.py", "--env", "backend")
@@ -411,7 +410,7 @@ def test_update(release: str) -> int:
 
 def test_acme(release: str, image: str) -> int:
     """Test ACMEv2 validation."""
-    info(f"Validating ACMVEv2 implementation {image}...")
+    info(f"Validating ACMVEv2 implementation on {image}...")
 
     compose_override = config.DEVSCRIPTS_FILES / "docker-compose.certbot.yaml"
     compose_files = f"docker-compose.yml:{compose_override}"
@@ -430,12 +429,7 @@ def test_acme(release: str, image: str) -> int:
 
         with utils.chdir(dest):
             # build containers
-            utils.run(
-                ["docker", "compose", "build", "--build-arg", f"IMAGE={image}"],
-                stderr=subprocess.DEVNULL,
-                stdout=subprocess.DEVNULL,
-                env=environ,
-            )
+            utils.run(["docker", "compose", "build", "--build-arg", f"IMAGE={image}"], env=environ)
 
             # Start containers
             with _compose_up(env=environ):
@@ -506,13 +500,16 @@ def validate_docker_compose_files(release: str) -> int:
 
 
 class Command(DevCommand):
+    """Class implementing the ``dev.py validate docker-compose`` command."""
+
     modules = (("django_ca", "django-ca"),)
     django_ca: ModuleType
     help_text = "Validate Docker Compose setup."
 
     @property
     def parser_parents(self) -> Sequence[argparse.ArgumentParser]:
-        return [self.parent.docker_options]  # type: ignore[attr-defined]  # set in the constructor
+        # pylint: disable-next=no-member  # set in the constructor of parent class
+        return [self.parent.docker_options]  # type: ignore[attr-defined]  # see pylint above
 
     def add_arguments(self, parser: argparse.ArgumentParser) -> None:
         parser.add_argument(

devscripts/validation/license_headers.py

@@ -73,6 +73,8 @@ def handle_python_file(path: Union[str, "os.PathLike[str]"], script: bool) -> in
 
 
 class Command(DevCommand):
+    """Class implementing the ``dev.py validate license-headers`` command."""
+
     help_text = "Ensure consistent license headers in source files."
 
     def handle(self, args: argparse.Namespace) -> None:

devscripts/validation/state.py

@@ -17,7 +17,6 @@
 import importlib.util
 import os
 import re
-import sys
 import types
 import typing
 from typing import Any, Dict, Union
@@ -250,6 +249,8 @@ def check_readme(project_config: Dict[str, Any]) -> int:
 
 
 class Command(DevCommand):
+    """Class implementing the ``dev.py validate state`` command."""
+
     help_text = "Validate state of various configuration and documentation files."
 
     def handle(self, args: argparse.Namespace) -> None: