From 64e2fe186ea318d10102215604beac0ae8848f2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Sch=C3=BCller?= Date: Tue, 26 Mar 2024 16:46:43 +0100 Subject: [PATCH] Makefile: implement "fully source containers" HMS-3883 --- .gitignore | 4 + Makefile | 72 ++++ .../osbuild-composer/entrypoint-onprem.py | 399 ++++++++++++++++++ containers/osbuild-composer/entrypoint.py | 78 +++- distribution/Dockerfile-composer | 37 ++ distribution/Dockerfile-composer_srcinstall | 38 ++ distribution/Dockerfile-fauxauth | 4 + distribution/Dockerfile-fauxauth_srcinstall | 25 ++ distribution/Dockerfile-ubi | 9 +- distribution/Dockerfile-worker | 12 +- distribution/Dockerfile-worker_srcinstall | 34 ++ distribution/osbuild-worker-entrypoint.sh | 13 + 12 files changed, 721 insertions(+), 4 deletions(-) create mode 100644 containers/osbuild-composer/entrypoint-onprem.py create mode 100644 distribution/Dockerfile-composer create mode 100644 distribution/Dockerfile-composer_srcinstall create mode 100644 distribution/Dockerfile-fauxauth_srcinstall create mode 100644 distribution/Dockerfile-worker_srcinstall create mode 100755 distribution/osbuild-worker-entrypoint.sh diff --git a/.gitignore b/.gitignore index cb05d4f4216..9bd117b8363 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,7 @@ __pycache__ /tools/appsre-ansible/inventory /docs/osbuild-composer.7 +go.local.mod +go.local.sum +container_worker_built.info +container_composer_built.info diff --git a/Makefile b/Makefile index fd6dcab89ec..6c7eb4e403e 100644 --- a/Makefile +++ b/Makefile @@ -155,6 +155,10 @@ install: build clean: rm -rf $(BUILDDIR)/bin/ rm -rf $(CURDIR)/rpmbuild + rm -rf $(BUILDDIR)/build/ + rm -f $(BUILDDIR)/go.local.* + rm -f $(BUILDDIR)/container_worker_built.info + rm -f $(BUILDDIR)/container_composer_built.info .PHONY: push-check push-check: build unit-tests srpm man @@ -274,3 +278,71 @@ scratch: $(RPM_SPECFILE) $(RPM_TARBALL) --nocheck \ $(RPM_SPECFILE) + +# either "docker" or "sudo podman" +# podman needs to build as root as it also needs to run as root afterwards +CONTAINER_EXECUTABLE ?= sudo podman + +DOCKER_IMAGE_WORKER := osbuild-worker_devel +DOCKERFILE_WORKER := distribution/Dockerfile-worker_srcinstall + +DOCKER_IMAGE_COMPOSER := osbuild-composer_devel +DOCKERFILE_COMPOSER := distribution/Dockerfile-composer_srcinstall + +GOPROXY ?= https://proxy.golang.org,direct + +# source where the other repos are locally +# has to end with a trailing slash +SRC_DEPS_EXTERNAL_CHECKOUT_DIR ?= ../ + +# names of folder that have to be git-cloned additionally to be able +# to build all code +SRC_DEPS_EXTERNAL_NAMES := images pulp-client +SRC_DEPS_EXTERNAL_DIRS := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR),$(SRC_DEPS_EXTERNAL_NAMES)) + +$(SRC_DEPS_EXTERNAL_DIRS): + @for DIR in $@; do if ! [ -d $$DIR ]; then echo "Please checkout $$DIR so it is available at $$DIR"; exit 1; fi; done + + +SRC_DEPS_DIRS := internal cmd pkg repositories + +# All files to check for rebuild! +SRC_DEPS := $(shell find $(SRC_DEPS_DIRS) -name *.go -or -name *.json) +SRC_DEPS_EXTERNAL := $(shell find $(SRC_DEPS_EXTERNAL_DIRS) -name *.go) + +# dependencies to rebuild worker +WORKER_SRC_DEPS := $(SRC_DEPS) +# dependencies to rebuild composer +COMPOSER_SRC_DEPS := $(SRC_DEPS) + +GOMODARGS ?= -modfile=go.local.mod + +USE_BTRFS ?= yes + +go.local.mod go.local.sum: $(SRC_DEPS_EXTERNAL_DIRS) go.mod $(SRC_DEPS_EXTERNAL) $(WORKER_SRC_DEPS) $(COMPOSER_SRC_DEPS) Makefile + cp go.mod go.local.mod + cp go.sum go.local.sum + + go mod edit $(GOMODARGS) -replace github.com/osbuild/images=$(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)images + go mod edit $(GOMODARGS) -replace github.com/osbuild/pulp-client=$(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)pulp-client + go mod edit $(GOMODARGS) -replace github.com/osbuild/osbuild-composer/pkg/splunk_logger=./pkg/splunk_logger + env GOPROXY=$(GOPROXY) go mod tidy $(GOMODARGS) + env GOPROXY=$(GOPROXY) go mod vendor $(GOMODARGS) + +container_worker_built.info: go.local.mod $(WORKER_SRC_DEPS) $(DOCKERFILE_WORKER) + $(CONTAINER_EXECUTABLE) build -t $(DOCKER_IMAGE_WORKER) -f $(DOCKERFILE_WORKER) --build-arg GOMODARGS=$(GOMODARGS) --build-arg USE_BTRFS=$(USE_BTRFS) . + echo "Worker last built on" > $@ + date >> $@ + +container_composer_built.info: go.local.mod $(COMPOSER_SRC_DEPS) $(DOCKERFILE_COMPOSER) + $(CONTAINER_EXECUTABLE) build -t $(DOCKER_IMAGE_COMPOSER) -f $(DOCKERFILE_COMPOSER) --build-arg GOMODARGS=$(GOMODARGS) . + echo "Composer last built on" > $@ + date >> $@ + +# build a container with a worker from full source +.PHONY: container_worker +container_worker: container_worker_built.info + +# build a container with the composer from full source +.PHONY: container_composer +container_composer: container_composer_built.info diff --git a/containers/osbuild-composer/entrypoint-onprem.py b/containers/osbuild-composer/entrypoint-onprem.py new file mode 100644 index 00000000000..d545d324b53 --- /dev/null +++ b/containers/osbuild-composer/entrypoint-onprem.py @@ -0,0 +1,399 @@ +"""entrypoint - Containerized OSBuild Composer + +This provides the entrypoint for a containerized osbuild-composer image. It +spawns `osbuild-composer` on start and manages it until it exits. The main +purpose of this entrypoint is to prepare everything to be usable from within +a container. +""" + +import argparse +import contextlib +import os +import pathlib +import signal +import socket +import subprocess +import sys + + +class Cli(contextlib.AbstractContextManager): + """Command Line Interface""" + + def __init__(self, argv): + self.args = None + self._argv = argv + self._exitstack = None + self._parser = None + + def _parse_args(self): + self._parser = argparse.ArgumentParser( + add_help=True, + allow_abbrev=False, + argument_default=None, + description="Containerized OSBuild Composer", + prog="container/osbuild-composer", + ) + + # --[no-]composer-api + self._parser.add_argument( + "--composer-api", + action="store_true", + dest="composer_api", + help="Enable the composer-API", + ) + self._parser.add_argument( + "--no-composer-api", + action="store_false", + dest="composer_api", + help="Disable the composer-API", + ) + self._parser.add_argument( + "--composer-api-port", + type=int, + default=8080, + dest="composer_api_port", + help="Port which the composer-API listens on", + ) + self._parser.add_argument( + "--composer-api-bind-address", + type=str, + default="::", + dest="composer_api_bind_address", + help="Bind the composer API to the specified address", + ) + + # --[no-]local-worker-api + self._parser.add_argument( + "--local-worker-api", + action="store_true", + dest="local_worker_api", + help="Enable the local-worker-API", + ) + self._parser.add_argument( + "--no-local-worker-api", + action="store_false", + dest="local_worker_api", + help="Disable the local-worker-API", + ) + + # --[no-]remote-worker-api + self._parser.add_argument( + "--remote-worker-api", + action="store_true", + dest="remote_worker_api", + help="Enable the remote-worker-API", + ) + self._parser.add_argument( + "--no-remote-worker-api", + action="store_false", + dest="remote_worker_api", + help="Disable the remote-worker-API", + ) + self._parser.add_argument( + "--remote-worker-api-port", + type=int, + default=8700, + dest="remote_worker_api_port", + help="Port which the remote-worker API listens on", + ) + self._parser.add_argument( + "--remote-worker-api-bind-address", + type=str, + default="::", + dest="remote_worker_api_bind_address", + help="Bind the remote worker API to the specified address", + ) + + # --[no-]weldr-api + self._parser.add_argument( + "--weldr-api", + action="store_true", + dest="weldr_api", + help="Enable the weldr-API", + ) + self._parser.add_argument( + "--no-weldr-api", + action="store_false", + dest="weldr_api", + help="Disable the weldr-API", + ) + + # --[no-]dnf-json + self._parser.add_argument( + "--dnf-json", + action="store_true", + dest="dnf_json", + help="Enable dnf-json", + ) + self._parser.add_argument( + "--no-dnf-json", + action="store_false", + dest="dnf_json", + help="Disable dnf-json", + ) + self._parser.add_argument( + "--dnf-json-port", + type=int, + default=0, + dest="dnf_json_port", + help="Specify the port dnf-json should listen on", + ) + + self._parser.set_defaults( + builtin_worker=False, + composer_api=False, + local_worker_api=False, + remote_worker_api=False, + weldr_api=False, + dnf_json=False, + ) + + return self._parser.parse_args(self._argv[1:]) + + def __enter__(self): + self._exitstack = contextlib.ExitStack() + self.args = self._parse_args() + return self + + def __exit__(self, exc_type, exc_value, exc_tb): + self._exitstack.close() + self._exitstack = None + + def _prepare_sockets(self): + # Prepare all the API sockets that osbuild-composer expectes, and make + # sure to pass them according to the systemd socket-activation API. + # + # Note that we rely on this being called early, so we get the correct + # FD numbers assigned. We need FD-#3 onwards for compatibility with + # socket activation (because python `subprocess.Popen` does not support + # renumbering the sockets we pass down). + + index = 3 + sockets = [] + names = [] + + # osbuild-composer.socket + if self.args.weldr_api: + print("Create weldr-api socket", file=sys.stderr) + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.bind("/run/weldr/api.socket") + sock.listen() + sockets.append(sock) + + names.append("osbuild-composer.socket") + + assert(sock.fileno() == index) + index += 1 + + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.bind("/run/cloudapi/api.socket") + sock.listen() + sockets.append(sock) + + names.append("osbuild-composer.socket") + + assert(sock.fileno() == index) + index += 1 + + # osbuild-composer-api.socket + if self.args.composer_api: + print("Create composer-api socket on port {}".format(self.args.composer_api_port) , file=sys.stderr) + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + sock.bind((self.args.composer_api_bind_address, self.args.composer_api_port)) + sock.listen() + sockets.append(sock) + names.append("osbuild-composer-api.socket") + + assert(sock.fileno() == index) + index += 1 + + # osbuild-local-worker.socket + if self.args.local_worker_api: + print("Create local-worker-api socket", file=sys.stderr) + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.bind("/run/osbuild-composer/job.socket") + sock.listen() + sockets.append(sock) + names.append("osbuild-local-worker.socket") + + assert(sock.fileno() == index) + index += 1 + + # osbuild-remote-worker.socket + if self.args.remote_worker_api: + print(f"Create remote-worker-api socket on address [{self.args.remote_worker_api_bind_address}]:{self.args.remote_worker_api_port}", file=sys.stderr) + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + sock.bind((self.args.remote_worker_api_bind_address, self.args.remote_worker_api_port)) + sock.listen(256) + sockets.append(sock) + names.append("osbuild-remote-worker.socket") + + assert(sock.fileno() == index) + index += 1 + + # Prepare FD environment for the child process. + os.environ["LISTEN_FDS"] = str(len(sockets)) + os.environ["LISTEN_FDNAMES"] = ":".join(names) + + return sockets + + @staticmethod + def _spawn_worker(): + cmd = [ + "/usr/libexec/osbuild-composer/osbuild-worker", + "-unix", + "/run/osbuild-composer/job.socket", + ] + + env = os.environ.copy() + env["CACHE_DIRECTORY"] = "/var/cache/osbuild-worker" + env["STATE_DIRECTORY"] = "/var/lib/osbuild-worker" + + return subprocess.Popen( + cmd, + cwd="/", + env=env, + stdin=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + ) + + @staticmethod + def _spawn_composer(sockets): + cmd = [ + "/usr/libexec/osbuild-composer/osbuild-composer", + "-verbose", + ] + + # Prepare the environment for osbuild-composer. Note that we cannot use + # the `env` parameter of `subprocess.Popen()`, because it conflicts + # with the `preexec_fn=` parameter. Therefore, we have to modify the + # caller's environment. + os.environ["CACHE_DIRECTORY"] = "/var/cache/osbuild-composer" + os.environ["STATE_DIRECTORY"] = "/var/lib/osbuild-composer" + + # We need to set `LISTEN_PID=` to the target PID. The only way python + # allows us to do this is to hook into `preexec_fn=`, which is executed + # by `subprocess.Popen()` after forking, but before executing the new + # executable. + preexec_setenv = lambda: os.putenv("LISTEN_PID", str(os.getpid())) + + return subprocess.Popen( + cmd, + cwd="/usr/libexec/osbuild-composer", + stdin=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + pass_fds=[sock.fileno() for sock in sockets], + preexec_fn=preexec_setenv, + ) + + def _spawn_dnf_json(self): + cmd = [ + "/usr/libexec/osbuild-depsolve-dnf", + ] + + if self.args.dnf_json_port: + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + sock.bind(("::", self.args.dnf_json_port)) + sock.listen() + else: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + os.makedirs("/run/osbuild-dnf-json/", exist_ok=True) + sock.bind("/run/osbuild-dnf-json/api.sock") + sock.listen() + + dnfenv = os.environ.copy() + dnfenv["LISTEN_FDS"] = "1" + dnfenv["LISTEN_FD"] = str(sock.fileno()) + + return subprocess.Popen( + cmd, + cwd="/usr/libexec/osbuild-composer", + stdin=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + env=dnfenv, + pass_fds=[sock.fileno()] + ) + + def run(self): + """Program Runtime""" + + proc_composer = None + proc_worker = None + proc_dnf_json = None + res = 0 + sockets = self._prepare_sockets() + + def handler(signum, frame): + proc_composer.terminate() + proc_worker.terminate() + proc_dnf_json.terminate() + + signal.signal(signal.SIGTERM, handler) + + liveness = pathlib.Path('/tmp/osbuild-composer-live') + + liveness.touch() + + try: + if self.args.builtin_worker: + proc_worker = self._spawn_worker() + + if self.args.dnf_json: + proc_dnf_json = self._spawn_dnf_json() + + if any([self.args.weldr_api, self.args.composer_api, self.args.local_worker_api, self.args.remote_worker_api]): + proc_composer = self._spawn_composer(sockets) + + if proc_composer: + res = proc_composer.wait() + + if proc_worker: + if proc_composer: + proc_worker.terminate() + proc_worker.wait() + + if proc_dnf_json: + if proc_composer: + proc_dnf_json.terminate() + proc_dnf_json.wait() + + except KeyboardInterrupt: + if proc_composer: + proc_composer.terminate() + res = proc_composer.wait() + if proc_worker: + proc_worker.terminate() + proc_worker.wait() + if proc_dnf_json: + proc_dnf_json.terminate() + proc_dnf_json.wait() + except: + if proc_worker: + proc_worker.kill() + if proc_dnf_json: + proc_dnf_json.kill() + if proc_composer: + proc_composer.kill() + raise + finally: + liveness.unlink() + + return res + + +if __name__ == "__main__": + with Cli(sys.argv) as global_main: + sys.exit(global_main.run()) diff --git a/containers/osbuild-composer/entrypoint.py b/containers/osbuild-composer/entrypoint.py index 8fc0b093b73..4bba6cfa4d7 100644 --- a/containers/osbuild-composer/entrypoint.py +++ b/containers/osbuild-composer/entrypoint.py @@ -153,6 +153,27 @@ def _parse_args(self): help="Disable the weldr-API", ) + # --[no-]dnf-json + self._parser.add_argument( + "--dnf-json", + action="store_true", + dest="dnf_json", + help="Enable dnf-json", + ) + self._parser.add_argument( + "--no-dnf-json", + action="store_false", + dest="dnf_json", + help="Disable dnf-json", + ) + self._parser.add_argument( + "--dnf-json-port", + type=int, + default=0, + dest="dnf_json_port", + help="Specify the port dnf-json should listen on", + ) + self._parser.set_defaults( builtin_worker=False, composer_api=False, @@ -160,6 +181,7 @@ def _parse_args(self): local_worker_api=False, remote_worker_api=False, weldr_api=False, + dnf_json=False, ) return self._parser.parse_args(self._argv[1:]) @@ -194,6 +216,18 @@ def _prepare_sockets(self): sock.bind("/run/weldr/api.socket") sock.listen() sockets.append(sock) + + names.append("osbuild-composer.socket") + + assert(sock.fileno() == index) + index += 1 + + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.bind("/run/cloudapi/api.socket") + sock.listen() + sockets.append(sock) + names.append("osbuild-composer.socket") assert(sock.fileno() == index) @@ -312,11 +346,44 @@ def _spawn_composer(sockets): preexec_fn=preexec_setenv, ) + def _spawn_dnf_json(self): + cmd = [ + "/usr/libexec/osbuild-depsolve-dnf", + ] + + if self.args.dnf_json_port: + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + sock.bind(("::", self.args.dnf_json_port)) + sock.listen() + else: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + os.makedirs("/run/osbuild-dnf-json/", exist_ok=True) + sock.bind("/run/osbuild-dnf-json/api.sock") + sock.listen() + + dnfenv = os.environ.copy() + dnfenv["LISTEN_FDS"] = "1" + dnfenv["LISTEN_FD"] = str(sock.fileno()) + + return subprocess.Popen( + cmd, + cwd="/usr/libexec/osbuild-composer", + stdin=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + env=dnfenv, + pass_fds=[sock.fileno()] + ) + def run(self): """Program Runtime""" proc_composer = None proc_worker = None + proc_dnf_json = None res = 0 sockets = self._prepare_sockets() @@ -325,6 +392,7 @@ def handler(signum, frame): time.sleep(self.args.shutdown_wait_period) proc_composer.terminate() proc_worker.terminate() + proc_dnf_json.terminate() signal.signal(signal.SIGTERM, handler) @@ -336,7 +404,10 @@ def handler(signum, frame): if self.args.builtin_worker: proc_worker = self._spawn_worker() - if any([self.args.weldr_api, self.args.composer_api, self.args.local_worker_api, self.args.remote_worker_api]): + if self.args.dnf_json: + proc_dnf_json = self._spawn_dnf_json() + + if should_launch_composer: proc_composer = self._spawn_composer(sockets) if proc_composer: @@ -354,9 +425,14 @@ def handler(signum, frame): if proc_worker: proc_worker.terminate() proc_worker.wait() + if proc_dnf_json: + proc_dnf_json.terminate() + proc_dnf_json.wait() except: if proc_worker: proc_worker.kill() + if proc_dnf_json: + proc_dnf_json.kill() if proc_composer: proc_composer.kill() raise diff --git a/distribution/Dockerfile-composer b/distribution/Dockerfile-composer new file mode 100644 index 00000000000..7975a5327a0 --- /dev/null +++ b/distribution/Dockerfile-composer @@ -0,0 +1,37 @@ +FROM fedora:39 AS builder +ENV GOBIN=/opt/app-root/src/go/bin + +RUN dnf install -y gpgme-devel libassuan-devel device-mapper-devel golang + +WORKDIR /osbuild-composer +COPY . /osbuild-composer +ENV GOFLAGS="-mod=vendor -tags=exclude_graphdriver_btrfs" + +ARG GOPROXY=https://proxy.golang.org,direct +RUN go env -w GOPROXY=$GOPROXY + +ARG GOMODARGS="" + +RUN go install $GOMODARGS ./cmd/osbuild-composer/ + +FROM registry.access.redhat.com/ubi9/go-toolset:latest AS builder2 +RUN go install github.com/jackc/tern@latest + +FROM fedora:39 + +RUN dnf install -y python3 python3-dnf gpgme libassuan device-mapper-libs +RUN mkdir -p "/usr/libexec/osbuild-composer" +RUN mkdir -p "/etc/osbuild-composer/" +RUN mkdir -p "/run/osbuild-composer/" +RUN mkdir -p "/var/cache/osbuild-composer/" +RUN mkdir -p "/var/lib/osbuild-composer/" +RUN mkdir -p "/usr/share/osbuild-composer/" +RUN mkdir -p "/opt/migrate/" +COPY --from=builder /opt/app-root/src/go/bin/osbuild-composer /usr/libexec/osbuild-composer/ +COPY ./containers/osbuild-composer/entrypoint.py /opt/entrypoint.py + +COPY ./pkg/jobqueue/dbjobqueue/schemas /opt/migrate/schemas +COPY --from=builder2 /opt/app-root/src/go/bin/tern /opt/migrate/ + +EXPOSE 8008 8080 8700 +ENTRYPOINT ["python3", "/opt/entrypoint.py", "--remote-worker-api", "--composer-api", "--prometheus", "--shutdown-wait-period", "15"] diff --git a/distribution/Dockerfile-composer_srcinstall b/distribution/Dockerfile-composer_srcinstall new file mode 100644 index 00000000000..d64d573ae1b --- /dev/null +++ b/distribution/Dockerfile-composer_srcinstall @@ -0,0 +1,38 @@ +FROM fedora:39 AS builder +ENV GOBIN=/opt/app-root/src/go/bin + +RUN dnf install -y gpgme-devel libassuan-devel device-mapper-devel golang + +WORKDIR /osbuild-composer +COPY . /osbuild-composer +ENV GOFLAGS="-mod=vendor -tags=exclude_graphdriver_btrfs" + +ARG GOPROXY=https://proxy.golang.org,direct +RUN go env -w GOPROXY=$GOPROXY + +ARG GOMODARGS="" +ARG GCFLAGS="" + +RUN go install $GOMODARGS $GCFLAGS ./cmd/osbuild-composer/ + +FROM registry.access.redhat.com/ubi9/go-toolset:latest AS builder2 +RUN go install github.com/jackc/tern@latest + +FROM osbuild_devel + +RUN dnf install -y python3 python3-dnf gpgme libassuan device-mapper-libs delve +RUN mkdir -p "/usr/libexec/osbuild-composer" +RUN mkdir -p "/etc/osbuild-composer/" +RUN mkdir -p "/run/osbuild-composer/" +RUN mkdir -p "/var/cache/osbuild-composer/" +RUN mkdir -p "/var/lib/osbuild-composer/" +RUN mkdir -p "/usr/share/osbuild-composer/" +RUN mkdir -p "/opt/migrate/" +COPY --from=builder /opt/app-root/src/go/bin/osbuild-composer /usr/libexec/osbuild-composer/ +COPY ./containers/osbuild-composer/entrypoint.py /opt/entrypoint.py + +COPY ./pkg/jobqueue/dbjobqueue/schemas /opt/migrate/schemas +COPY --from=builder2 /opt/app-root/src/go/bin/tern /opt/migrate/ + +EXPOSE 8008 8080 8700 +ENTRYPOINT ["python3", "/opt/entrypoint.py", "--remote-worker-api", "--composer-api", "--prometheus", "--shutdown-wait-period", "15"] diff --git a/distribution/Dockerfile-fauxauth b/distribution/Dockerfile-fauxauth index 5bb23e47735..52c423f6273 100644 --- a/distribution/Dockerfile-fauxauth +++ b/distribution/Dockerfile-fauxauth @@ -4,6 +4,10 @@ FROM registry.access.redhat.com/ubi9/go-toolset:latest AS builder # a repository owned by a different user. COPY --chown=1001 . . ENV GOFLAGS=-mod=vendor + +ARG GOPROXY=https://proxy.golang.org,direct +RUN go env -w GOPROXY=$GOPROXY + RUN go install ./cmd/osbuild-mock-openid-provider/ FROM registry.access.redhat.com/ubi9/ubi-minimal:latest diff --git a/distribution/Dockerfile-fauxauth_srcinstall b/distribution/Dockerfile-fauxauth_srcinstall new file mode 100644 index 00000000000..ef6ed4adff4 --- /dev/null +++ b/distribution/Dockerfile-fauxauth_srcinstall @@ -0,0 +1,25 @@ +FROM fedora:39 AS builder +ENV GOBIN=/opt/app-root/src/go/bin + +RUN dnf install -y gpgme-devel libassuan-devel device-mapper-devel golang + +COPY . . +ENV GOFLAGS=-mod=vendor + +ARG GOPROXY=https://proxy.golang.org,direct +RUN go env -w GOPROXY=$GOPROXY + +ARG GOMODARGS="" + +RUN go install $GOMODARGS ./cmd/osbuild-mock-openid-provider/ + +FROM registry.access.redhat.com/ubi9/ubi-minimal:latest +RUN microdnf install -y python3 +RUN mkdir -p "/usr/libexec/osbuild-composer" +RUN mkdir -p "/etc/osbuild-composer/" + +COPY --from=builder /opt/app-root/src/go/bin/osbuild-mock-openid-provider /usr/libexec/osbuild-composer/ +COPY ./containers/fauxauth/fauxauth.py /opt/fauxauth.py + +EXPOSE 8080 8080 +ENTRYPOINT "/opt/fauxauth.py" diff --git a/distribution/Dockerfile-ubi b/distribution/Dockerfile-ubi index d17c2ee562e..2fc20dc4935 100644 --- a/distribution/Dockerfile-ubi +++ b/distribution/Dockerfile-ubi @@ -13,11 +13,18 @@ ARG COMMIT ENV LDFLAGS="${COMMIT:+-X \'github.com/osbuild/osbuild-composer/internal/common.GitRev=${COMMIT}\'}" ENV LDFLAGS="${LDFLAGS:+-ldflags=\"${LDFLAGS}\"}" ENV GOFLAGS="-mod=vendor -tags=exclude_graphdriver_btrfs" + +ARG GOPROXY=https://proxy.golang.org,direct +RUN go env -w GOPROXY=$GOPROXY + +ARG GOMODARGS="" +ENV GOMODARGS=$GOMODARGS + # if run without "sh -c", podman for some reason executes the command in a way, # which results in the following error: # [1/3] STEP 12/12: RUN go install ${LDFLAGS} ./cmd/osbuild-composer/ # invalid value "\"-X" for flag -ldflags: missing = in = -RUN /usr/bin/sh -c "go install ${LDFLAGS} ./cmd/osbuild-composer/" +RUN /usr/bin/sh -c "go install ${LDFLAGS} ${GOMODARGS} ./cmd/osbuild-composer/" FROM registry.access.redhat.com/ubi9/go-toolset:latest AS builder2 RUN go install github.com/jackc/tern@latest diff --git a/distribution/Dockerfile-worker b/distribution/Dockerfile-worker index a35e36e1c59..87901bbf9bd 100644 --- a/distribution/Dockerfile-worker +++ b/distribution/Dockerfile-worker @@ -1,4 +1,4 @@ -FROM fedora:37 AS builder +FROM fedora:39 AS builder ENV GOBIN=/opt/app-root/src/go/bin # extra packages are needed # to compile osbuild @@ -6,14 +6,22 @@ RUN dnf install -y golang \ krb5-devel \ gpgme-devel \ libassuan-devel + +ARG USE_BTRFS=no +RUN if [[ "$USE_BTRFS" == "yes" ]]; then dnf install -y btrfs-progs-devel device-mapper-devel; fi + WORKDIR /osbuild-composer COPY . /osbuild-composer ENV GOFLAGS=-mod=vendor + +ARG GOPROXY=https://proxy.golang.org,direct +RUN go env -w GOPROXY=$GOPROXY + RUN go install ./cmd/osbuild-worker FROM fedora RUN dnf update -y && dnf upgrade -y -RUN dnf install -y libxcrypt-compat qemu-img osbuild osbuild-ostree +RUN dnf install -y libxcrypt-compat qemu-img osbuild osbuild-ostree osbuild-depsolve-dnf RUN mkdir -p "/usr/libexec/osbuild-composer" RUN mkdir -p "/etc/osbuild-composer/" RUN mkdir -p "/run/osbuild-composer/" diff --git a/distribution/Dockerfile-worker_srcinstall b/distribution/Dockerfile-worker_srcinstall new file mode 100644 index 00000000000..6eb78d4ffe2 --- /dev/null +++ b/distribution/Dockerfile-worker_srcinstall @@ -0,0 +1,34 @@ +FROM fedora:39 AS builder +ENV GOBIN=/opt/app-root/src/go/bin +# extra packages are needed +# to compile osbuild +RUN dnf install -y golang \ + krb5-devel \ + gpgme-devel \ + libassuan-devel + +ARG USE_BTRFS=no +RUN if [[ "$USE_BTRFS" == "yes" ]]; then dnf install -y btrfs-progs-devel device-mapper-devel; fi + +WORKDIR /osbuild-composer +COPY . /osbuild-composer +ENV GOFLAGS=-mod=vendor + +ARG GOPROXY=https://proxy.golang.org,direct +RUN go env -w GOPROXY=$GOPROXY + +ARG GOMODARGS="" + +RUN go install $GOMODARGS ./cmd/osbuild-worker + +FROM osbuild_devel + +RUN mkdir -p "/usr/libexec/osbuild-composer" +RUN mkdir -p "/etc/osbuild-composer/" +RUN mkdir -p "/run/osbuild-composer/" +RUN mkdir -p "/var/cache/osbuild-worker/" +RUN mkdir -p "/var/lib/osbuild-composer/" +RUN mkdir -p "/var/cache/osbuild-composer/output" +COPY --from=builder /opt/app-root/src/go/bin/osbuild-worker /usr/libexec/osbuild-composer/ + +ENTRYPOINT ["/usr/libexec/osbuild-composer/osbuild-worker"] diff --git a/distribution/osbuild-worker-entrypoint.sh b/distribution/osbuild-worker-entrypoint.sh new file mode 100755 index 00000000000..40d338ad069 --- /dev/null +++ b/distribution/osbuild-worker-entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +APP="/usr/libexec/osbuild-composer/osbuild-worker" +APP_ARGS="${WORKER_ARGS:-}" + +if [[ -n "${GODEBUG_PORT:-}" ]]; then + echo "With golang debugger enabled on port ${GODEBUG_PORT} ..." + echo "NOTE: you HAVE to attach the debugger NOW otherwise the osbuild-worker will not continue running" + /usr/bin/dlv "--listen=:${GODEBUG_PORT}" --headless=true --api-version=2 exec ${APP} -- ${APP_ARGS} + exit $? +fi + +${APP} ${APP_ARGS}