From e6c4e8efd92361b29972066705c26ec8392a73a3 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Thu, 13 Aug 2026 21:59:08 +0000 Subject: [PATCH 1/5] Carry buildkit's environment, daemon flags, and socket to the builder The builder wraps buildkit and exposed almost none of it: two color variables crossed from the caller's environment into the container, the daemon's flags were unreachable, and its socket never left the VM. Every BUILDKIT_ variable in the caller's environment now rides into the builder, so BUILDKIT_HOST points the shim's client at a daemon of the operator's choosing, the BUILDKIT_TLS set carries that address's credentials, and whatever buildkit documents next needs no new plumbing; NO_COLOR rides along as the conventional outlier the color handling already honored, and a change to any of them recreates the builder the way the color variables always did. --buildkitd-flags takes one string of daemon flags, buildx's own contract for the same name, handed to the shim after -- where it passes them to buildkitd verbatim. --publish-buildkit-socket puts the daemon's socket on the host through the runtime's socket publishing, where buildctl and buildx dial it directly, the daemon-socket convention Docker Desktop, colima, and podman machine follow on macOS; remote machines reach the same socket through buildkit's own ssh scheme. --- .../Builder/BuilderStart.swift | 68 +++++++++++++++++-- 1 file changed, 61 insertions(+), 7 deletions(-) diff --git a/Sources/ContainerCommands/Builder/BuilderStart.swift b/Sources/ContainerCommands/Builder/BuilderStart.swift index f8bf20972..0f1ea4c61 100644 --- a/Sources/ContainerCommands/Builder/BuilderStart.swift +++ b/Sources/ContainerCommands/Builder/BuilderStart.swift @@ -26,6 +26,7 @@ import ContainerizationExtras import ContainerizationOCI import Foundation import Logging +import SystemPackage import TerminalProgress extension Application { @@ -46,6 +47,24 @@ extension Application { ) var memory: String? + @Option( + name: .long, + help: ArgumentHelp( + """ + Flags for the buildkitd the builder launches, one string the way buildx's \ + --buildkitd-flags takes them. They begin with a dash, which is how an option \ + begins, so the value attaches to the option: --buildkitd-flags=--debug + """, + valueName: "flags")) + var buildkitdFlags: String? + + @Option( + name: .customLong("publish-buildkit-socket"), + help: ArgumentHelp( + "Publish the builder's buildkitd socket to this host path, where buildctl and buildx reach it directly", + valueName: "path")) + var publishBuildkitSocket: String? + @OptionGroup public var dns: Flags.DNS @@ -70,6 +89,8 @@ extension Application { cpus: self.cpus, memory: self.memory, log: log, + buildkitdFlags: self.buildkitdFlags, + publishBuildkitSocket: self.publishBuildkitSocket, dnsNameservers: self.dns.nameservers, dnsDomain: self.dns.domain, dnsSearchDomains: self.dns.searchDomains, @@ -85,6 +106,8 @@ extension Application { memory: String?, log: Logger, ssh: Bool = false, + buildkitdFlags: String? = nil, + publishBuildkitSocket: String? = nil, dnsNameservers: [String] = [], dnsDomain: String? = nil, dnsSearchDomains: [String] = [], @@ -115,9 +138,16 @@ extension Application { let builderPlatform = ContainerizationOCI.Platform(arch: "arm64", os: "linux", variant: "v8") + // Every buildkit variable in the caller's environment rides into + // the builder: BUILDKIT_HOST points the shim at a daemon of the + // operator's choosing, the BUILDKIT_TLS set carries that + // address's credentials, the color settings shape progress + // output, and whatever buildkit documents next needs no new + // plumbing here. NO_COLOR rides along as the conventional + // outlier the color handling already honored. var targetEnvVars: [String] = [] - if let buildkitColors = ProcessInfo.processInfo.environment["BUILDKIT_COLORS"] { - targetEnvVars.append("BUILDKIT_COLORS=\(buildkitColors)") + for (name, value) in ProcessInfo.processInfo.environment where name.hasPrefix("BUILDKIT_") { + targetEnvVars.append("\(name)=\(value)") } if ProcessInfo.processInfo.environment["NO_COLOR"] != nil { targetEnvVars.append("NO_COLOR=true") @@ -142,11 +172,16 @@ extension Application { let existingDNS = existingContainer.configuration.dns let existingManagedEnv = existingEnv.filter { envVar in - envVar.hasPrefix("BUILDKIT_COLORS=") || envVar.hasPrefix("NO_COLOR=") + envVar.hasPrefix("BUILDKIT_") || envVar.hasPrefix("NO_COLOR=") }.sorted() let envChanged = existingManagedEnv != targetEnvVars + let existingPublished = existingContainer.configuration.publishedSockets + .map { "\($0.containerPath):\($0.hostPath)" }.sorted() + let targetPublished = (publishBuildkitSocket.map { ["/run/buildkit/buildkitd.sock:\($0)"] } ?? []).sorted() + let publishChanged = existingPublished != targetPublished + // Check if we need to recreate the builder due to different image let imageChanged = existingImage != builderImage let cpuChanged = existingResources.cpus != resources.cpus @@ -172,8 +207,8 @@ extension Application { switch existingContainer.status { case .running: - guard imageChanged || cpuChanged || memChanged || envChanged || dnsChanged || sshChanged else { - // If image, mem, cpu, env, and DNS are the same, continue using the existing builder + guard imageChanged || cpuChanged || memChanged || envChanged || dnsChanged || sshChanged || publishChanged else { + // If image, mem, cpu, env, DNS, and published sockets are the same, continue using the existing builder return } // If they changed, stop and delete the existing builder @@ -182,7 +217,7 @@ extension Application { case .stopped: // If the builder is stopped and matches our requirements, start it // Otherwise, delete it and create a new one - if imageChanged || cpuChanged || memChanged || envChanged || dnsChanged || sshChanged { + if imageChanged || cpuChanged || memChanged || envChanged || dnsChanged || sshChanged || publishChanged { try? await client.delete(id: existingContainer.id) } else { do { @@ -209,11 +244,17 @@ extension Application { } let useRosetta = containerSystemConfig.build.rosetta - let shimArguments = [ + var shimArguments = [ "--debug", "--vsock", useRosetta ? nil : "--enable-qemu", ].compactMap { $0 } + if let buildkitdFlags { + // One string of daemon flags, buildx's own contract for + // --buildkitd-flags, handed to the shim after -- where it + // passes them to buildkitd verbatim. + shimArguments += ["--"] + buildkitdFlags.split(separator: " ").map(String.init) + } guard ManagedContainer.nameValid(Builder.builderContainerId) else { throw ContainerizationError(.invalidArgument, message: "container ID \(Builder.builderContainerId) is not a valid container ID") @@ -258,6 +299,19 @@ extension Application { var config = ContainerConfiguration(id: Builder.builderContainerId, image: imageDesc, process: processConfig) config.resources = resources config.ssh = ssh && ProcessInfo.processInfo.environment["SSH_AUTH_SOCK"] != nil + if let publishBuildkitSocket { + // buildkitd listens at its library default inside the VM; + // publishing that socket puts a file on the host where + // buildctl and buildx dial the daemon directly, the + // daemon-socket convention Docker Desktop, colima, and + // podman machine follow on macOS. + config.publishedSockets = [ + try PublishSocket( + containerPath: FilePath("/run/buildkit/buildkitd.sock"), + hostPath: FilePath(publishBuildkitSocket) + ) + ] + } config.labels = [ ResourceLabelKeys.plugin: "builder", ResourceLabelKeys.role: ResourceRoleValues.builder, From 8e9ca58bcbc92a1ac06a7a7f5b1e4cb562b9123d Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Thu, 13 Aug 2026 23:44:44 +0000 Subject: [PATCH 2/5] Test the builder's buildkit exposure knobs end to end One builder start carries a BUILDKIT_ environment variable, daemon flags after --buildkitd-flags, and --publish-buildkit-socket. The test reads each where its consumer does: the shim's environment at /proc/1/environ, buildkitd's argv for the passed flags, and the host filesystem for the published socket, then runs a build with all three applied. --- .../Build/TestCLIBuilderExposeSerial.swift | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 Tests/IntegrationTests/Build/TestCLIBuilderExposeSerial.swift diff --git a/Tests/IntegrationTests/Build/TestCLIBuilderExposeSerial.swift b/Tests/IntegrationTests/Build/TestCLIBuilderExposeSerial.swift new file mode 100644 index 000000000..d83cdc0b5 --- /dev/null +++ b/Tests/IntegrationTests/Build/TestCLIBuilderExposeSerial.swift @@ -0,0 +1,96 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerTestSupport +import Foundation +import Testing + +/// Tests for the builder's buildkit exposure knobs: environment +/// inheritance, daemon flag passthrough, and socket publishing. +/// +/// Serialized because starting and deleting the shared `buildkit` +/// container would race with in-flight builds in the concurrent pool. +@Suite(.serialized) +struct TestCLIBuilderExposeSerial { + + /// One builder start carries all three knobs; each lands where its + /// consumer reads it. + @Test func testBuilderStartCarriesEnvFlagsAndPublishedSocket() async throws { + try await ContainerFixture.with { f in + f.addCleanup { try? f.builderDelete(force: true) } + + let marker = "echo-\(f.testID)" + let sockDir = try f.makeShortSocketDir("bk") + let hostSocket = "\(sockDir)/buildkitd.sock" + + try f.run( + [ + "builder", "start", + // The value begins with a dash, so it attaches to the + // option rather than following it as a separate word. + "--buildkitd-flags=--debug", + "--publish-buildkit-socket", hostSocket, + ], + env: ["BUILDKIT_TEST_MARKER": marker] + ).check() + try await f.waitForBuilderRunning() + + // Every BUILDKIT_* variable in the CLI's environment rides into + // the builder's init process, where the shim reads it. + let environ = try f.doExec( + "buildkit", cmd: ["sh", "-c", "tr '\\0' '\\n' < /proc/1/environ"]) + #expect( + environ.contains("BUILDKIT_TEST_MARKER=\(marker)"), + "a BUILDKIT_ variable in the caller's environment reaches the shim: \(environ)") + + // The published daemon socket appears as a file on the host once + // the relay is attached. + var socketAppeared = false + for _ in 0..<50 { + if FileManager.default.fileExists(atPath: hostSocket) { + socketAppeared = true + break + } + try await Task.sleep(for: .milliseconds(100)) + } + #expect(socketAppeared, "the published buildkitd socket exists at \(hostSocket)") + + // Post-separator flags reach buildkitd's own argv verbatim. + var cmdline = "" + for _ in 0..<20 { + cmdline = + (try? f.doExec( + "buildkit", + cmd: ["sh", "-c", "xargs -0 < /proc/$(pidof -s buildkitd)/cmdline"])) ?? "" + if !cmdline.isEmpty { break } + try await Task.sleep(for: .milliseconds(500)) + } + #expect( + cmdline.contains("buildkitd") && cmdline.contains("--debug"), + "buildkitd runs with the passed daemon flags: \(cmdline)") + + // The builder serves builds with all three knobs applied. + let dir = try f.createTempDir() + try f.createContext( + dir: dir, + dockerfile: """ + FROM scratch + ENV SEED=expose + """) + try f.build(tag: "test-expose:\(f.testID)", contextDir: dir) + } + } +} From 2a69d342240fa2cf1f78c685187df3dcaa83d3a7 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 14 Aug 2026 00:18:32 +0000 Subject: [PATCH 3/5] Stamp the builder's injected environment on its record The builder's process environment mixes the image's own variables with the ones start injects; moby/buildkit's image carries BUILDKIT_SETUP_CGROUPV2_ROOT, so a start that reads the whole environment back sees variables it never injected and recreates a builder that already matches its target, and concurrent builds then race one another's delete and create cycles. The injected set rides a label on the container record, stamped at create and compared on the next start, the way docker compose stamps com.docker.compose.config-hash on the services it manages. --- .../Builder/BuilderStart.swift | 19 ++++++++++++++----- .../Common/ResourceLabels.swift | 5 +++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Sources/ContainerCommands/Builder/BuilderStart.swift b/Sources/ContainerCommands/Builder/BuilderStart.swift index 0f1ea4c61..e9f2e9290 100644 --- a/Sources/ContainerCommands/Builder/BuilderStart.swift +++ b/Sources/ContainerCommands/Builder/BuilderStart.swift @@ -168,14 +168,19 @@ extension Application { if let existingContainer { let existingImage = existingContainer.configuration.image.reference let existingResources = existingContainer.configuration.resources - let existingEnv = existingContainer.configuration.initProcess.environment let existingDNS = existingContainer.configuration.dns - let existingManagedEnv = existingEnv.filter { envVar in - envVar.hasPrefix("BUILDKIT_") || envVar.hasPrefix("NO_COLOR=") - }.sorted() + // The record's process environment mixes the image's own + // variables with the ones this start injects (the image may + // carry BUILDKIT_ variables of its own, as moby/buildkit's + // does), so the injected set is read from the label stamped + // at create, the way docker compose stamps + // com.docker.compose.config-hash on the services it manages. + // https://github.com/docker/compose/blob/main/pkg/api/labels.go + let existingManagedEnv = existingContainer.configuration.labels[ResourceLabelKeys.builderEnvironment] + .flatMap { try? JSONDecoder().decode([String].self, from: Data($0.utf8)) } - let envChanged = existingManagedEnv != targetEnvVars + let envChanged = (existingManagedEnv ?? []) != targetEnvVars let existingPublished = existingContainer.configuration.publishedSockets .map { "\($0.containerPath):\($0.hostPath)" }.sorted() @@ -315,6 +320,10 @@ extension Application { config.labels = [ ResourceLabelKeys.plugin: "builder", ResourceLabelKeys.role: ResourceRoleValues.builder, + // The injected environment, stamped so a later start can + // compare its target against the variables it manages. + ResourceLabelKeys.builderEnvironment: String( + decoding: try JSONEncoder().encode(targetEnvVars), as: UTF8.self), ] config.capAdd = ["ALL"] config.mounts = [ diff --git a/Sources/ContainerResource/Common/ResourceLabels.swift b/Sources/ContainerResource/Common/ResourceLabels.swift index b02564a04..7089449e6 100644 --- a/Sources/ContainerResource/Common/ResourceLabels.swift +++ b/Sources/ContainerResource/Common/ResourceLabels.swift @@ -105,6 +105,11 @@ public struct ResourceLabelKeys { /// Indicates a resource with a reserved or dedicated purpose. public static let role = "com.apple.container.resource.role" + + /// Records the environment variables a builder start injected into the + /// builder, distinguishing them from variables the image's own + /// configuration carries. + public static let builderEnvironment = "com.apple.container.builder.environment" } /// System-defined values for resource the resource role label. From ebd4407053b9328c5ceffa97612587a8bc5d30c5 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 14 Aug 2026 00:21:54 +0000 Subject: [PATCH 4/5] Test that builder start keeps a matching builder and recreates on change A start whose inputs match the running builder returns it untouched; a start whose managed environment differs replaces it. A file written into the builder's filesystem tells the cases apart: it survives a kept builder and is absent from a recreated one. --- .../Build/TestCLIBuilderExposeSerial.swift | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/Tests/IntegrationTests/Build/TestCLIBuilderExposeSerial.swift b/Tests/IntegrationTests/Build/TestCLIBuilderExposeSerial.swift index d83cdc0b5..7e20a0444 100644 --- a/Tests/IntegrationTests/Build/TestCLIBuilderExposeSerial.swift +++ b/Tests/IntegrationTests/Build/TestCLIBuilderExposeSerial.swift @@ -93,4 +93,31 @@ struct TestCLIBuilderExposeSerial { try f.build(tag: "test-expose:\(f.testID)", contextDir: dir) } } + + /// A second start with identical knobs keeps the running builder; a + /// changed managed environment recreates it. Observed through a file + /// in the builder's filesystem: a kept builder still has it, a + /// recreated one boots without it. + @Test func testBuilderStartIsStableUntilItsInputsChange() async throws { + try await ContainerFixture.with { f in + f.addCleanup { try? f.builderDelete(force: true) } + + let marker = "echo-\(f.testID)" + try f.run(["builder", "start"], env: ["BUILDKIT_TEST_MARKER": marker]).check() + try await f.waitForBuilderRunning() + _ = try f.doExec("buildkit", cmd: ["touch", "/tmp/witness"]) + + try f.run(["builder", "start"], env: ["BUILDKIT_TEST_MARKER": marker]).check() + try await f.waitForBuilderRunning() + let kept = try f.doExec( + "buildkit", cmd: ["sh", "-c", "ls /tmp/witness 2>/dev/null || echo gone"]) + #expect(kept.contains("/tmp/witness"), "an unchanged start keeps the running builder: \(kept)") + + try f.run(["builder", "start"], env: ["BUILDKIT_TEST_MARKER": "\(marker)-changed"]).check() + try await f.waitForBuilderRunning() + let recreated = try f.doExec( + "buildkit", cmd: ["sh", "-c", "ls /tmp/witness 2>/dev/null || echo gone"]) + #expect(recreated.contains("gone"), "a changed environment recreates the builder: \(recreated)") + } + } } From 13bb8ee907ccf1b1aaeea5203a7b5f80ade57ae3 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 14 Aug 2026 01:10:55 +0000 Subject: [PATCH 5/5] Say which settings drifted when replacing the builder A builder replaced mid-flight interrupts every build sharing it, so the replacement names its reason: the drifted settings print to stderr beside the progress output before the stop. --- Sources/ContainerCommands/Builder/BuilderStart.swift | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Sources/ContainerCommands/Builder/BuilderStart.swift b/Sources/ContainerCommands/Builder/BuilderStart.swift index e9f2e9290..f7213d6fb 100644 --- a/Sources/ContainerCommands/Builder/BuilderStart.swift +++ b/Sources/ContainerCommands/Builder/BuilderStart.swift @@ -210,6 +210,14 @@ extension Application { return false }() + // The drifted settings, named so a replaced builder says why + // on stderr, beside the progress output. + let drifted: [String] = [ + ("image", imageChanged), ("cpus", cpuChanged), ("memory", memChanged), + ("environment", envChanged), ("dns", dnsChanged), ("ssh", sshChanged), + ("published sockets", publishChanged), + ].filter(\.1).map(\.0) + switch existingContainer.status { case .running: guard imageChanged || cpuChanged || memChanged || envChanged || dnsChanged || sshChanged || publishChanged else { @@ -217,12 +225,16 @@ extension Application { return } // If they changed, stop and delete the existing builder + FileHandle.standardError.write( + Data("recreating builder: \(drifted.joined(separator: ", ")) changed\n".utf8)) try await client.stop(id: existingContainer.id) try await client.delete(id: existingContainer.id) case .stopped: // If the builder is stopped and matches our requirements, start it // Otherwise, delete it and create a new one if imageChanged || cpuChanged || memChanged || envChanged || dnsChanged || sshChanged || publishChanged { + FileHandle.standardError.write( + Data("recreating builder: \(drifted.joined(separator: ", ")) changed\n".utf8)) try? await client.delete(id: existingContainer.id) } else { do {