diff --git a/Sources/ContainerCommands/Builder/BuilderStart.swift b/Sources/ContainerCommands/Builder/BuilderStart.swift index f8bf20972..f7213d6fb 100644 --- a/Sources/ContainerCommands/Builder/BuilderStart.swift +++ b/Sources/ContainerCommands/Builder/BuilderStart.swift @@ -26,6 +26,7 @@ import ContainerizationExtras import ContainerizationOCI import Foundation import Logging +import SystemPackage import TerminalProgress extension Application { @@ -46,6 +47,24 @@ extension Application { ) var memory: String? + @Option( + name: .long, + help: ArgumentHelp( + """ + Flags for the buildkitd the builder launches, one string the way buildx's \ + --buildkitd-flags takes them. They begin with a dash, which is how an option \ + begins, so the value attaches to the option: --buildkitd-flags=--debug + """, + valueName: "flags")) + var buildkitdFlags: String? + + @Option( + name: .customLong("publish-buildkit-socket"), + help: ArgumentHelp( + "Publish the builder's buildkitd socket to this host path, where buildctl and buildx reach it directly", + valueName: "path")) + var publishBuildkitSocket: String? + @OptionGroup public var dns: Flags.DNS @@ -70,6 +89,8 @@ extension Application { cpus: self.cpus, memory: self.memory, log: log, + buildkitdFlags: self.buildkitdFlags, + publishBuildkitSocket: self.publishBuildkitSocket, dnsNameservers: self.dns.nameservers, dnsDomain: self.dns.domain, dnsSearchDomains: self.dns.searchDomains, @@ -85,6 +106,8 @@ extension Application { memory: String?, log: Logger, ssh: Bool = false, + buildkitdFlags: String? = nil, + publishBuildkitSocket: String? = nil, dnsNameservers: [String] = [], dnsDomain: String? = nil, dnsSearchDomains: [String] = [], @@ -115,9 +138,16 @@ extension Application { let builderPlatform = ContainerizationOCI.Platform(arch: "arm64", os: "linux", variant: "v8") + // Every buildkit variable in the caller's environment rides into + // the builder: BUILDKIT_HOST points the shim at a daemon of the + // operator's choosing, the BUILDKIT_TLS set carries that + // address's credentials, the color settings shape progress + // output, and whatever buildkit documents next needs no new + // plumbing here. NO_COLOR rides along as the conventional + // outlier the color handling already honored. var targetEnvVars: [String] = [] - if let buildkitColors = ProcessInfo.processInfo.environment["BUILDKIT_COLORS"] { - targetEnvVars.append("BUILDKIT_COLORS=\(buildkitColors)") + for (name, value) in ProcessInfo.processInfo.environment where name.hasPrefix("BUILDKIT_") { + targetEnvVars.append("\(name)=\(value)") } if ProcessInfo.processInfo.environment["NO_COLOR"] != nil { targetEnvVars.append("NO_COLOR=true") @@ -138,14 +168,24 @@ extension Application { if let existingContainer { let existingImage = existingContainer.configuration.image.reference let existingResources = existingContainer.configuration.resources - let existingEnv = existingContainer.configuration.initProcess.environment let existingDNS = existingContainer.configuration.dns - let existingManagedEnv = existingEnv.filter { envVar in - envVar.hasPrefix("BUILDKIT_COLORS=") || envVar.hasPrefix("NO_COLOR=") - }.sorted() + // The record's process environment mixes the image's own + // variables with the ones this start injects (the image may + // carry BUILDKIT_ variables of its own, as moby/buildkit's + // does), so the injected set is read from the label stamped + // at create, the way docker compose stamps + // com.docker.compose.config-hash on the services it manages. + // https://github.com/docker/compose/blob/main/pkg/api/labels.go + let existingManagedEnv = existingContainer.configuration.labels[ResourceLabelKeys.builderEnvironment] + .flatMap { try? JSONDecoder().decode([String].self, from: Data($0.utf8)) } + + let envChanged = (existingManagedEnv ?? []) != targetEnvVars - let envChanged = existingManagedEnv != targetEnvVars + let existingPublished = existingContainer.configuration.publishedSockets + .map { "\($0.containerPath):\($0.hostPath)" }.sorted() + let targetPublished = (publishBuildkitSocket.map { ["/run/buildkit/buildkitd.sock:\($0)"] } ?? []).sorted() + let publishChanged = existingPublished != targetPublished // Check if we need to recreate the builder due to different image let imageChanged = existingImage != builderImage @@ -170,19 +210,31 @@ extension Application { return false }() + // The drifted settings, named so a replaced builder says why + // on stderr, beside the progress output. + let drifted: [String] = [ + ("image", imageChanged), ("cpus", cpuChanged), ("memory", memChanged), + ("environment", envChanged), ("dns", dnsChanged), ("ssh", sshChanged), + ("published sockets", publishChanged), + ].filter(\.1).map(\.0) + switch existingContainer.status { case .running: - guard imageChanged || cpuChanged || memChanged || envChanged || dnsChanged || sshChanged else { - // If image, mem, cpu, env, and DNS are the same, continue using the existing builder + guard imageChanged || cpuChanged || memChanged || envChanged || dnsChanged || sshChanged || publishChanged else { + // If image, mem, cpu, env, DNS, and published sockets are the same, continue using the existing builder return } // If they changed, stop and delete the existing builder + FileHandle.standardError.write( + Data("recreating builder: \(drifted.joined(separator: ", ")) changed\n".utf8)) try await client.stop(id: existingContainer.id) try await client.delete(id: existingContainer.id) case .stopped: // If the builder is stopped and matches our requirements, start it // Otherwise, delete it and create a new one - if imageChanged || cpuChanged || memChanged || envChanged || dnsChanged || sshChanged { + if imageChanged || cpuChanged || memChanged || envChanged || dnsChanged || sshChanged || publishChanged { + FileHandle.standardError.write( + Data("recreating builder: \(drifted.joined(separator: ", ")) changed\n".utf8)) try? await client.delete(id: existingContainer.id) } else { do { @@ -209,11 +261,17 @@ extension Application { } let useRosetta = containerSystemConfig.build.rosetta - let shimArguments = [ + var shimArguments = [ "--debug", "--vsock", useRosetta ? nil : "--enable-qemu", ].compactMap { $0 } + if let buildkitdFlags { + // One string of daemon flags, buildx's own contract for + // --buildkitd-flags, handed to the shim after -- where it + // passes them to buildkitd verbatim. + shimArguments += ["--"] + buildkitdFlags.split(separator: " ").map(String.init) + } guard ManagedContainer.nameValid(Builder.builderContainerId) else { throw ContainerizationError(.invalidArgument, message: "container ID \(Builder.builderContainerId) is not a valid container ID") @@ -258,9 +316,26 @@ extension Application { var config = ContainerConfiguration(id: Builder.builderContainerId, image: imageDesc, process: processConfig) config.resources = resources config.ssh = ssh && ProcessInfo.processInfo.environment["SSH_AUTH_SOCK"] != nil + if let publishBuildkitSocket { + // buildkitd listens at its library default inside the VM; + // publishing that socket puts a file on the host where + // buildctl and buildx dial the daemon directly, the + // daemon-socket convention Docker Desktop, colima, and + // podman machine follow on macOS. + config.publishedSockets = [ + try PublishSocket( + containerPath: FilePath("/run/buildkit/buildkitd.sock"), + hostPath: FilePath(publishBuildkitSocket) + ) + ] + } config.labels = [ ResourceLabelKeys.plugin: "builder", ResourceLabelKeys.role: ResourceRoleValues.builder, + // The injected environment, stamped so a later start can + // compare its target against the variables it manages. + ResourceLabelKeys.builderEnvironment: String( + decoding: try JSONEncoder().encode(targetEnvVars), as: UTF8.self), ] config.capAdd = ["ALL"] config.mounts = [ diff --git a/Sources/ContainerResource/Common/ResourceLabels.swift b/Sources/ContainerResource/Common/ResourceLabels.swift index b02564a04..7089449e6 100644 --- a/Sources/ContainerResource/Common/ResourceLabels.swift +++ b/Sources/ContainerResource/Common/ResourceLabels.swift @@ -105,6 +105,11 @@ public struct ResourceLabelKeys { /// Indicates a resource with a reserved or dedicated purpose. public static let role = "com.apple.container.resource.role" + + /// Records the environment variables a builder start injected into the + /// builder, distinguishing them from variables the image's own + /// configuration carries. + public static let builderEnvironment = "com.apple.container.builder.environment" } /// System-defined values for resource the resource role label. diff --git a/Tests/IntegrationTests/Build/TestCLIBuilderExposeSerial.swift b/Tests/IntegrationTests/Build/TestCLIBuilderExposeSerial.swift new file mode 100644 index 000000000..7e20a0444 --- /dev/null +++ b/Tests/IntegrationTests/Build/TestCLIBuilderExposeSerial.swift @@ -0,0 +1,123 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerTestSupport +import Foundation +import Testing + +/// Tests for the builder's buildkit exposure knobs: environment +/// inheritance, daemon flag passthrough, and socket publishing. +/// +/// Serialized because starting and deleting the shared `buildkit` +/// container would race with in-flight builds in the concurrent pool. +@Suite(.serialized) +struct TestCLIBuilderExposeSerial { + + /// One builder start carries all three knobs; each lands where its + /// consumer reads it. + @Test func testBuilderStartCarriesEnvFlagsAndPublishedSocket() async throws { + try await ContainerFixture.with { f in + f.addCleanup { try? f.builderDelete(force: true) } + + let marker = "echo-\(f.testID)" + let sockDir = try f.makeShortSocketDir("bk") + let hostSocket = "\(sockDir)/buildkitd.sock" + + try f.run( + [ + "builder", "start", + // The value begins with a dash, so it attaches to the + // option rather than following it as a separate word. + "--buildkitd-flags=--debug", + "--publish-buildkit-socket", hostSocket, + ], + env: ["BUILDKIT_TEST_MARKER": marker] + ).check() + try await f.waitForBuilderRunning() + + // Every BUILDKIT_* variable in the CLI's environment rides into + // the builder's init process, where the shim reads it. + let environ = try f.doExec( + "buildkit", cmd: ["sh", "-c", "tr '\\0' '\\n' < /proc/1/environ"]) + #expect( + environ.contains("BUILDKIT_TEST_MARKER=\(marker)"), + "a BUILDKIT_ variable in the caller's environment reaches the shim: \(environ)") + + // The published daemon socket appears as a file on the host once + // the relay is attached. + var socketAppeared = false + for _ in 0..<50 { + if FileManager.default.fileExists(atPath: hostSocket) { + socketAppeared = true + break + } + try await Task.sleep(for: .milliseconds(100)) + } + #expect(socketAppeared, "the published buildkitd socket exists at \(hostSocket)") + + // Post-separator flags reach buildkitd's own argv verbatim. + var cmdline = "" + for _ in 0..<20 { + cmdline = + (try? f.doExec( + "buildkit", + cmd: ["sh", "-c", "xargs -0 < /proc/$(pidof -s buildkitd)/cmdline"])) ?? "" + if !cmdline.isEmpty { break } + try await Task.sleep(for: .milliseconds(500)) + } + #expect( + cmdline.contains("buildkitd") && cmdline.contains("--debug"), + "buildkitd runs with the passed daemon flags: \(cmdline)") + + // The builder serves builds with all three knobs applied. + let dir = try f.createTempDir() + try f.createContext( + dir: dir, + dockerfile: """ + FROM scratch + ENV SEED=expose + """) + try f.build(tag: "test-expose:\(f.testID)", contextDir: dir) + } + } + + /// A second start with identical knobs keeps the running builder; a + /// changed managed environment recreates it. Observed through a file + /// in the builder's filesystem: a kept builder still has it, a + /// recreated one boots without it. + @Test func testBuilderStartIsStableUntilItsInputsChange() async throws { + try await ContainerFixture.with { f in + f.addCleanup { try? f.builderDelete(force: true) } + + let marker = "echo-\(f.testID)" + try f.run(["builder", "start"], env: ["BUILDKIT_TEST_MARKER": marker]).check() + try await f.waitForBuilderRunning() + _ = try f.doExec("buildkit", cmd: ["touch", "/tmp/witness"]) + + try f.run(["builder", "start"], env: ["BUILDKIT_TEST_MARKER": marker]).check() + try await f.waitForBuilderRunning() + let kept = try f.doExec( + "buildkit", cmd: ["sh", "-c", "ls /tmp/witness 2>/dev/null || echo gone"]) + #expect(kept.contains("/tmp/witness"), "an unchanged start keeps the running builder: \(kept)") + + try f.run(["builder", "start"], env: ["BUILDKIT_TEST_MARKER": "\(marker)-changed"]).check() + try await f.waitForBuilderRunning() + let recreated = try f.doExec( + "buildkit", cmd: ["sh", "-c", "ls /tmp/witness 2>/dev/null || echo gone"]) + #expect(recreated.contains("gone"), "a changed environment recreates the builder: \(recreated)") + } + } +}