feat(sftkit): add devel command for building debian packages with dh …

…virtualenv

sftkit/sftkit/devel/_cli.py

@@ -1,12 +1,14 @@
 import asyncio
+import json
 from types import SimpleNamespace
-from typing import Annotated
+from typing import Annotated, Optional
 
 import typer
 
 from sftkit import database
-from sftkit.devel._config import read_config
 from sftkit.util import log_setup
+from ._config import read_config
+from ._debian import build_debian_packages as _build_debian_packages
 
 cli = typer.Typer()
 
@@ -21,14 +23,59 @@ def get_config(
     log_setup(verbose - quiet)
     asyncio.get_event_loop().set_debug(debug)
 
-    config = read_config()
-    ctx.obj = SimpleNamespace(config=config)
+    project_root, pyproject_toml, config = read_config()
+    ctx.obj = SimpleNamespace(pyproject_toml=pyproject_toml, sftkit_config=config, project_root=project_root)
 
 
 @cli.command()
 def create_migration(ctx: typer.Context, name: str):
     """Create a new database migration"""
-    database.create_migration(ctx.obj.config.db_migrations_dir, name)
+    database.create_migration(ctx.obj.sftkit_config.db_migrations_dir, name)
+
+
+@cli.command()
+def build_debian_packages(
+    ctx: typer.Context,
+    jobs: Annotated[int, typer.Option("--jobs", "-j", help="specify the number of builds to run in parallel")] = 1,
+    no_check: Annotated[bool, typer.Option("--no-check", help="skip running tests after building")] = False,
+    docker_executable: Annotated[
+        str, typer.Option("--docker-executable", help="path to the docker executable")
+    ] = "docker",
+    docker_build_args: Annotated[
+        Optional[list[str]], typer.Option("--docker-build-arg", help="arguments to pass to the underlying docker build")
+    ] = None,
+    dists: Annotated[Optional[list[str]], typer.Argument(help="a list of distributions to build for")] = None,
+):
+    """Build debian packages in docker container runners"""
+    target_distros = dists or ctx.obj.sftkit_config.target_debian_distros
+    if target_distros is None:
+        print("No target distributions were configured in the [tool.sftkit] section in the pyproject.toml")
+        raise typer.Exit(1)
+
+    project_name = ctx.obj.pyproject_toml.project.name
+
+    _build_debian_packages(
+        project_name=project_name,
+        jobs=jobs,
+        distributions=target_distros,
+        no_check=no_check,
+        docker_executable=docker_executable,
+        docker_build_args=docker_build_args or [],
+        project_root=ctx.obj.project_root,
+    )
+
+
+@cli.command()
+def list_debian_distros(
+    ctx: typer.Context,
+):
+    """List configured debian package targets"""
+    target_distros = ctx.obj.sftkit_config.target_debian_distros
+    if target_distros is None:
+        print("No target distributions were configured in the [tool.sftkit] section in the pyproject.toml")
+        raise typer.Exit(1)
+
+    print(json.dumps(ctx.obj.sftkit_config.target_debian_distros))
 
 
 def cli_main():

sftkit/sftkit/devel/_config.py

@@ -5,10 +5,60 @@
 from pydantic import BaseModel
 
 
+class BuildSystem(BaseModel):
+    requires: list[str] = []
+    build_backend: str | None = None
+    backend_path: list[str] = []
+
+
+class Readme(BaseModel):
+    file: str | None = None
+    text: str | None = None
+    content_type: str | None = None
+
+
+class License(BaseModel):
+    file: str | None = None
+    text: str | None = None
+
+
+class Contributor(BaseModel):
+    name: str | None = None
+    email: str | None = None
+
+
+class ProjectConfig(BaseModel):
+    name: str | None = None
+    version: str | None = None
+    description: str | None = None
+    readme: str | Readme | None = None
+    license: str | License | None = None
+    authors: list[Contributor] = []
+    maintainers: list[Contributor] = []
+    keywords: list[str] = []
+    classifiers: list[str] = []
+    urls: dict[str, str] = {}
+    requires_python: str | None = None
+    dependencies: list[str] = []
+    optional_dependencies: dict[str, list[str]] = {}
+    scripts: dict[str, str] = {}
+    gui_scripts: dict[str, str] = {}
+    entry_points: dict[str, dict[str, str]] = {}
+    dynamic: list[str] = []
+
+
+class PyprojectTOML(BaseModel):
+    build_system: BuildSystem | None = None
+    project: ProjectConfig | None = None
+    tool: dict[str, dict[str, Any]] = {}
+
+
 class SftkitDevelConfig(BaseModel):
     db_code_dir: Path
     db_migrations_dir: Path
 
+    target_debian_distros: list[str] | None = None
+
 
 def _load_toml(path: Path) -> dict[str, Any]:
     with open(path, "rb") as f:
@@ -26,16 +76,18 @@ def _find_pyproject_toml(start_path: Path | None) -> Path | None:
     return None
 
 
-def _parse_pyproject_toml(pyproject_toml_path: Path) -> SftkitDevelConfig:
+def _parse_pyproject_toml(pyproject_toml_path: Path) -> tuple[PyprojectTOML, SftkitDevelConfig]:
     pyproject_toml = _load_toml(pyproject_toml_path)
-    config: dict[str, Any] = pyproject_toml.get("tool", {}).get("sftkit", {})
-    return SftkitDevelConfig.model_validate(config)
+    pyproject_config = PyprojectTOML.model_validate(pyproject_toml)
+    sftkit_config: dict[str, Any] = pyproject_toml.get("tool", {}).get("sftkit", {})
+    sftkit_parsed = SftkitDevelConfig.model_validate(sftkit_config)
+    return pyproject_config, sftkit_parsed
 
 
-def read_config(search_path: Path | None = None) -> SftkitDevelConfig:
+def read_config(search_path: Path | None = None) -> tuple[Path, PyprojectTOML, SftkitDevelConfig]:
     start_path = search_path or Path.cwd()
     pyproject_path = _find_pyproject_toml(start_path)
     if pyproject_path is None:
         raise ValueError(f"Could not find a pyproject.toml file in any directory above {start_path}")
-    parsed = _parse_pyproject_toml(pyproject_path)
-    return parsed
+    pyproject_toml, sftkit_config = _parse_pyproject_toml(pyproject_path)
+    return pyproject_path.parent, pyproject_toml, sftkit_config

sftkit/sftkit/devel/_debian.py

@@ -0,0 +1,192 @@
+#!/usr/bin/python3
+
+# Build the Debian packages using Docker images.
+#
+# This script builds the Docker images and then executes them sequentially, each
+# one building a Debian package for the targeted operating system. It is
+# designed to be a "single command" to produce all the images.
+#
+# By default, builds for all known distributions, but a list of distributions
+# can be passed on the commandline for debugging.
+
+# Taken from https://github.com/matrix-org/synapse/blob/develop/debian/build_virtualenv, released under Apache 2
+
+import os
+import signal
+import subprocess
+import sys
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import Optional, Sequence
+
+DISTS = (
+    "debian:bookworm",
+    "debian:trixie",
+)
+
+
+class Builder:
+    def __init__(
+        self,
+        project_root: Path,
+        project_name: str,
+        docker_executable: str,
+        redirect_stdout=False,
+        docker_build_args: Optional[Sequence[str]] = None,
+    ):
+        self.project_root = project_root
+        self.project_name = project_name
+        self.docker_executable = docker_executable
+        self.redirect_stdout = redirect_stdout
+        self._docker_build_args = tuple(docker_build_args or ())
+        self.active_containers: set[str] = set()
+        self._lock = threading.Lock()
+        self._failed = False
+
+    def run_build(self, dist: str, skip_tests=False):
+        """Build deb for a single distribution"""
+
+        if self._failed:
+            print("not building %s due to earlier failure" % (dist,))
+            raise RuntimeError("failed")
+
+        try:
+            self._inner_build(dist, skip_tests)
+        except Exception as e:
+            print("build of %s failed: %s" % (dist, e), file=sys.stderr)
+            self._failed = True
+            raise
+
+    def _inner_build(self, dist: str, skip_tests=False):
+        tag = dist.split(":", 1)[1]
+
+        # Make the dir where the debs will live.
+        #
+        # Note that we deliberately put this outside the source tree, otherwise
+        # we tend to get source packages which are full of debs. (We could hack
+        # around that with more magic in the build_debian.sh script, but that
+        # doesn't solve the problem for natively-run dpkg-buildpakage).
+        debsdir = os.path.join(self.project_root, "../debs")
+        os.makedirs(debsdir, exist_ok=True)
+
+        if self.redirect_stdout:
+            logfile = os.path.join(debsdir, "%s.buildlog" % (tag,))
+            print("building %s: directing output to %s" % (dist, logfile))
+            stdout = open(logfile, "w", encoding="utf-8")
+        else:
+            stdout = None
+
+        # first build a docker image for the build environment
+        build_args = (
+            (
+                self.docker_executable,
+                "build",
+                "--tag",
+                "dh-venv-builder:" + tag,
+                "--build-arg",
+                "distro=" + dist,
+                "-f",
+                "docker/dhvirtualenv.Dockerfile",
+            )
+            + self._docker_build_args
+            + (".",)
+        )
+
+        subprocess.check_call(
+            build_args,
+            stdout=stdout,
+            stderr=subprocess.STDOUT,
+            cwd=self.project_root,
+        )
+
+        container_name = f"{self.project_name}_build_{tag}"
+        with self._lock:
+            self.active_containers.add(container_name)
+
+        # then run the build itself
+        subprocess.check_call(
+            [
+                self.docker_executable,
+                "run",
+                "--rm",
+                "--name",
+                container_name,
+                "--volume=" + debsdir + ":/debs",
+                "-e",
+                "TARGET_USERID=%i" % (os.getuid(),),
+                "-e",
+                "TARGET_GROUPID=%i" % (os.getgid(),),
+                "-e",
+                "DEB_BUILD_OPTIONS=%s" % ("nocheck" if skip_tests else ""),
+                "dh-venv-builder:" + tag,
+            ],
+            stdout=stdout,
+            stderr=subprocess.STDOUT,
+        )
+
+        with self._lock:
+            self.active_containers.remove(container_name)
+
+        if stdout is not None:
+            stdout.close()
+            print("Completed build of %s" % (dist,))
+
+    def kill_containers(self):
+        with self._lock:
+            active = list(self.active_containers)
+
+        for c in active:
+            print("killing container %s" % (c,))
+            subprocess.run(
+                [
+                    self.docker_executable,
+                    "kill",
+                    c,
+                ],
+                stdout=subprocess.DEVNULL,
+                check=True,
+            )
+            with self._lock:
+                self.active_containers.remove(c)
+
+
+def run_builds(builder: Builder, dists, jobs=1, skip_tests=False):
+    def sig(signum, _frame):
+        del signum  # unused
+
+        print("Caught SIGINT")
+        builder.kill_containers()
+
+    signal.signal(signal.SIGINT, sig)
+
+    with ThreadPoolExecutor(max_workers=jobs) as e:
+        res = e.map(lambda dist: builder.run_build(dist, skip_tests), dists)
+
+    # make sure we consume the iterable so that exceptions are raised.
+    for _ in res:
+        pass
+
+
+def build_debian_packages(
+    project_root: Path,
+    project_name: str,
+    distributions: list[str],
+    jobs: int,
+    docker_executable: str,
+    no_check: bool,
+    docker_build_args: list[str],
+):
+    global_builder = Builder(
+        project_root=project_root,
+        project_name=project_name,
+        docker_executable=docker_executable,
+        redirect_stdout=jobs > 1,
+        docker_build_args=docker_build_args,
+    )
+    run_builds(
+        global_builder,
+        dists=distributions,
+        jobs=jobs,
+        skip_tests=no_check,
+    )