diff --git a/Sources/ContainerBuild/BuildAPI+Extensions.swift b/Sources/ContainerBuild/BuildAPI+Extensions.swift index b052b1042..c91416d6a 100644 --- a/Sources/ContainerBuild/BuildAPI+Extensions.swift +++ b/Sources/ContainerBuild/BuildAPI+Extensions.swift @@ -36,6 +36,14 @@ extension BuildTransfer { return method == "" ? nil : method } + /// The BuildKit local-dir name this transfer belongs to: empty or + /// "context" for the primary build context, otherwise the name of a + /// `--build-context` the CLI declared. + func dirName() -> String? { + let dirName = self.metadata["dir-name"] + return dirName == "" ? nil : dirName + } + func includePatterns() -> [String]? { guard let includePatternsString = self.metadata["include-patterns"] else { return nil diff --git a/Sources/ContainerBuild/BuildFSSync.swift b/Sources/ContainerBuild/BuildFSSync.swift index c5a5288f2..27dccece7 100644 --- a/Sources/ContainerBuild/BuildFSSync.swift +++ b/Sources/ContainerBuild/BuildFSSync.swift @@ -55,7 +55,13 @@ import GRPCCore actor BuildFSSync: BuildPipelineHandler { let contextDir: URL - init(_ contextDir: URL) throws { + /// Local `--build-context` directories by name. The host is the sole + /// authority for this mapping: the shim only ever names a context, and a + /// name outside this map is refused, so the builder VM cannot steer the + /// host toward a path that was never declared on the command line. + let namedContexts: [String: URL] + + init(_ contextDir: URL, namedContexts: [String: URL] = [:]) throws { let resolved = contextDir.resolvingSymlinksInPath() guard FileManager.default.fileExists(atPath: contextDir.cleanPath) else { throw Error.contextNotFound(contextDir.cleanPath) @@ -64,7 +70,42 @@ actor BuildFSSync: BuildPipelineHandler { throw Error.contextIsNotDirectory(contextDir.cleanPath) } + var resolvedNamed: [String: URL] = [:] + for (name, dir) in namedContexts { + let resolvedDir = dir.resolvingSymlinksInPath() + guard FileManager.default.fileExists(atPath: dir.cleanPath) else { + throw Error.contextNotFound(dir.cleanPath) + } + guard resolvedDir.isDirectory else { + throw Error.contextIsNotDirectory(dir.cleanPath) + } + resolvedNamed[name] = resolvedDir + } + self.contextDir = resolved + self.namedContexts = resolvedNamed + } + + /// Resolve the context root a transfer operates on from the dir-name the + /// shim relayed. Every serving path (walk, read, info) resolves through + /// here, so the same containment enforcement applies to every root. + /// + /// This mirrors BuildKit's own session provider, which resolves the + /// `dir-name` metadata through a registered directory source and answers + /// NotFound for a name it does not hold, so a name is the only thing a + /// requester can choose and the server keeps the paths. The primary + /// context and the dockerfile directory arrive under BuildKit's default + /// names for them. + /// https://github.com/moby/buildkit/blob/v0.29.0/session/filesync/filesync.go + /// https://github.com/moby/buildkit/blob/v0.29.0/frontend/dockerui/context.go + private func contextRoot(_ packet: BuildTransfer) throws -> URL { + guard let name = packet.dirName(), name != "context", name != "dockerfile" else { + return self.contextDir + } + guard let root = self.namedContexts[name] else { + throw Error.unknownNamedContext(name) + } + return root } nonisolated func accept(_ packet: ServerStream) throws -> Bool { @@ -102,22 +143,23 @@ actor BuildFSSync: BuildPipelineHandler { func read(_ sender: AsyncStream.Continuation, _ packet: BuildTransfer, _ buildID: String) async throws { let offset: UInt64 = packet.offset() ?? 0 let size: Int = packet.len() ?? 0 + let root = try contextRoot(packet) var path: URL if packet.source.hasPrefix("/") { path = URL(fileURLWithPath: packet.source).standardizedFileURL } else { path = - contextDir + root .appendingPathComponent(packet.source) .standardizedFileURL } if !FileManager.default.fileExists(atPath: path.cleanPath) { - path = URL(filePath: self.contextDir.cleanPath) + path = URL(filePath: root.cleanPath) path.append(components: packet.source.cleanPathComponent) } let resolved = path.resolvingSymlinksInPath() - guard self.contextDir.parentOf(resolved) else { - throw Error.pathIsNotChild(resolved.cleanPath, self.contextDir.cleanPath) + guard root.parentOf(resolved) else { + throw Error.pathIsNotChild(resolved.cleanPath, root.cleanPath) } let data = try { if try path.isDir() { @@ -127,7 +169,7 @@ actor BuildFSSync: BuildPipelineHandler { return try file.data(offset: offset, length: size) ?? Data() }() - let transfer = try path.buildTransfer(id: packet.id, contextDir: self.contextDir, complete: true, data: data) + let transfer = try path.buildTransfer(id: packet.id, contextDir: root, complete: true, data: data) var response = ClientStream() response.buildID = buildID response.buildTransfer = transfer @@ -142,20 +184,21 @@ actor BuildFSSync: BuildPipelineHandler { /// normal `Walk`-based build. Must reject paths that escape the context root /// via symlinks for the same reasons as ``read(_:_:_:)``. func info(_ sender: AsyncStream.Continuation, _ packet: BuildTransfer, _ buildID: String) async throws { + let root = try contextRoot(packet) let path: URL if packet.source.hasPrefix("/") { path = URL(fileURLWithPath: packet.source).standardizedFileURL } else { path = - contextDir + root .appendingPathComponent(packet.source) .standardizedFileURL } let resolved = path.resolvingSymlinksInPath() - guard self.contextDir.parentOf(resolved) else { - throw Error.pathIsNotChild(resolved.cleanPath, self.contextDir.cleanPath) + guard root.parentOf(resolved) else { + throw Error.pathIsNotChild(resolved.cleanPath, root.cleanPath) } - let transfer = try path.buildTransfer(id: packet.id, contextDir: self.contextDir, complete: true) + let transfer = try path.buildTransfer(id: packet.id, contextDir: root, complete: true) var response = ClientStream() response.buildID = buildID response.buildTransfer = transfer @@ -200,30 +243,32 @@ actor BuildFSSync: BuildPipelineHandler { _ buildID: String ) async throws { let wantsTar = packet.mode() == "tar" + let root = try contextRoot(packet) var entries: [String: Set] = [:] let followPaths: [String] = packet.followPaths() ?? [] - let followPathsWalked = try walk(root: self.contextDir, includePatterns: followPaths) + let followPathsWalked = try walk(root: root, includePatterns: followPaths) + for url in followPathsWalked { - guard self.contextDir.absoluteURL.cleanPath != url.absoluteURL.cleanPath else { + guard root.absoluteURL.cleanPath != url.absoluteURL.cleanPath else { continue } - guard self.contextDir.parentOf(url) else { + guard root.parentOf(url) else { continue } - let relPath = try url.relativeChildPath(to: contextDir) - let parentPath = try url.deletingLastPathComponent().relativeChildPath(to: contextDir) + let relPath = try url.relativeChildPath(to: root) + let parentPath = try url.deletingLastPathComponent().relativeChildPath(to: root) let entry = DirEntry(url: url, isDirectory: url.hasDirectoryPath, relativePath: relPath) entries[parentPath, default: []].insert(entry) if url.isSymlink { let target: URL = url.resolvingSymlinksInPath() - if self.contextDir.parentOf(target) { - let relPath = try target.relativeChildPath(to: self.contextDir) + if root.parentOf(target) { + let relPath = try target.relativeChildPath(to: root) let entry = DirEntry(url: target, isDirectory: target.hasDirectoryPath, relativePath: relPath) - let parentPath: String = try target.deletingLastPathComponent().relativeChildPath(to: self.contextDir) + let parentPath: String = try target.deletingLastPathComponent().relativeChildPath(to: root) entries[parentPath, default: []].insert(entry) } } @@ -234,7 +279,7 @@ actor BuildFSSync: BuildPipelineHandler { if !wantsTar { let fileInfos = try fileOrder.map { rel -> FileInfo in - try FileInfo(path: contextDir.appendingPathComponent(rel), contextDir: contextDir) + try FileInfo(path: root.appendingPathComponent(rel), contextDir: root) } let data = try JSONEncoder().encode(fileInfos) @@ -268,15 +313,15 @@ actor BuildFSSync: BuildPipelineHandler { filter: .none) let tarHash = try Archiver.compress( - source: contextDir, + source: root, destination: tarURL, writerConfiguration: writerCfg ) { url in - guard let rel = try? url.relativeChildPath(to: contextDir) else { + guard let rel = try? url.relativeChildPath(to: root) else { return nil } - guard let parent = try? url.deletingLastPathComponent().relativeChildPath(to: self.contextDir) else { + guard let parent = try? url.deletingLastPathComponent().relativeChildPath(to: root) else { return nil } @@ -450,6 +495,7 @@ extension BuildFSSync { case couldNotDetermineUID(String) case couldNotDetermineGID(String) case pathIsNotChild(String, String) + case unknownNamedContext(String) var description: String { switch self { @@ -477,6 +523,8 @@ extension BuildFSSync { return "could not determine GID of file at path: \(path)" case .pathIsNotChild(let path, let parent): return "\(path) is not a child of \(parent)" + case .unknownNamedContext(let name): + return "no --build-context named \(name) was declared" } } } diff --git a/Sources/ContainerBuild/BuildImageResolver.swift b/Sources/ContainerBuild/BuildImageResolver.swift index 5b15352d8..9768aba3a 100644 --- a/Sources/ContainerBuild/BuildImageResolver.swift +++ b/Sources/ContainerBuild/BuildImageResolver.swift @@ -17,6 +17,7 @@ import ContainerAPIClient import ContainerPersistence import Containerization +import ContainerizationArchive import ContainerizationOCI import Foundation import GRPCCore @@ -85,6 +86,12 @@ struct BuildImageResolver: BuildPipelineHandler { let progress = ProgressBar(config: progressConfig) defer { progress.finish() } progress.start() + if let parsed = parseLocalRef(ref: ref) { + guard let image = try await handleLocalUCI(url: parsed.url, digest: parsed.digest) else { + throw Error.imageNotFound + } + return image + } if self.pull { return try await ClientImage.pull(reference: ref, platform: platform, containerSystemConfig: containerSystemConfig, progressUpdate: progress.handler) @@ -121,6 +128,79 @@ struct BuildImageResolver: BuildPipelineHandler { } throw Error.unknownPlatformForImage(platform.description, ref) } + + // handle oci-layout from build-context + // Not throwing here so that we can fall back to pull or fetch + private func parseLocalRef(ref: String) -> (url: URL, digest: String?)? { + guard let range = ref.firstRange(of: "oci-layout://") else { + return nil + } + var ref = ref + ref.removeSubrange(range) + + let digest = extractDigest(ref) + if let digest, let range = ref.range(of: digest) { + ref.removeSubrange(range) + } + ref = ref.trimmingCharacters(in: ["@"]) + if !FileManager.default.fileExists(atPath: ref) { + return nil + } + let url = URL(filePath: ref) + guard url.isDirectory else { + return nil + } + return (url, digest) + } + + private func handleLocalUCI(url: URL, digest: String?) async throws -> ClientImage? { + let tarURL = URL.temporaryDirectory + .appendingPathComponent(UUID().uuidString + ".tar") + + defer { try? FileManager.default.removeItem(at: tarURL) } + + let writerCfg = ArchiveWriterConfiguration( + format: .paxRestricted, + filter: .none + ) + + let root = url + + let _ = try Archiver.compress( + source: url, + destination: tarURL, + writerConfiguration: writerCfg, + closure: { url in + guard let rel = try? url.relativeChildPath(to: root) else { + return nil + } + return Archiver.ArchiveEntryInfo( + pathOnHost: url, + pathInArchive: URL(fileURLWithPath: rel) + ) + } + ) + + let result = try await ClientImage.load(from: tarURL.absolutePath(), force: true) + let images = result.images + let first = images.first + if let digest { + return images.first(where: { $0.digest == digest }) ?? first + } + return first + } + + private func extractDigest(_ string: String) -> String? { + guard let reg = try? Regex("[A-Za-z][A-Za-z0-9]*(?:[-_+.][A-Za-z][A-Za-z0-9]*)*[:][[:xdigit:]]{32,}") else { + return nil + } + let matches = string.matches(of: reg) + // digest always comes last when specified + guard let last = matches.last else { + return nil + } + return String(string[last.range]) + } } extension ImageTransfer { diff --git a/Sources/ContainerBuild/BuildPipelineHandler.swift b/Sources/ContainerBuild/BuildPipelineHandler.swift index da0b0081e..124fc6ef6 100644 --- a/Sources/ContainerBuild/BuildPipelineHandler.swift +++ b/Sources/ContainerBuild/BuildPipelineHandler.swift @@ -76,9 +76,21 @@ protocol BuildPipelineHandler: Sendable { public actor BuildPipeline { let handlers: [BuildPipelineHandler] public init(_ config: Builder.BuildConfig) async throws { + // Local named contexts are the `name=/absolute/path` entries; the CLI + // resolved every local value to an absolute path when it validated the + // flag, so a leading slash is what distinguishes a directory from an + // image, git, URL or oci-layout reference here. + var namedContexts: [String: URL] = [:] + for entry in config.buildContexts { + let parts = entry.split(separator: "=", maxSplits: 1) + guard parts.count == 2, parts[1].hasPrefix("/") else { + continue + } + namedContexts[String(parts[0])] = URL(filePath: String(parts[1])) + } self.handlers = [ - try BuildFSSync(URL(filePath: config.contextDir)), + try BuildFSSync(URL(filePath: config.contextDir), namedContexts: namedContexts), try BuildRemoteContentProxy(config.contentStore), try BuildImageResolver( config.contentStore, diff --git a/Sources/ContainerBuild/Builder.swift b/Sources/ContainerBuild/Builder.swift index 151ee033e..79f7a1084 100644 --- a/Sources/ContainerBuild/Builder.swift +++ b/Sources/ContainerBuild/Builder.swift @@ -267,6 +267,13 @@ public struct Builder: Sendable { public let buildID: String public let contentStore: ContentStore public let buildArgs: [String] + public let buildContexts: [String] + public let addHosts: [String] + public let ulimits: [String] + public let hostname: String? + public let shmSize: UInt64? + public let cgroupParent: String? + public let network: String? public let secrets: [String: Data] public let ssh: String public let contextDir: String @@ -289,6 +296,13 @@ public struct Builder: Sendable { buildID: String, contentStore: ContentStore, buildArgs: [String], + buildContexts: [String], + addHosts: [String] = [], + ulimits: [String] = [], + hostname: String? = nil, + shmSize: UInt64? = nil, + cgroupParent: String? = nil, + network: String? = nil, secrets: [String: Data], ssh: String, contextDir: String, @@ -310,6 +324,13 @@ public struct Builder: Sendable { self.buildID = buildID self.contentStore = contentStore self.buildArgs = buildArgs + self.buildContexts = buildContexts + self.addHosts = addHosts + self.ulimits = ulimits + self.hostname = hostname + self.shmSize = shmSize + self.cgroupParent = cgroupParent + self.network = network self.secrets = secrets self.ssh = ssh self.contextDir = contextDir @@ -356,6 +377,27 @@ public struct Builder: Sendable { for buildArg in config.buildArgs { metadata.addString(buildArg, forKey: "build-args") } + for buildContext in config.buildContexts { + metadata.addString(buildContext, forKey: "build-context") + } + for addHost in config.addHosts { + metadata.addString(addHost, forKey: "add-host") + } + for ulimit in config.ulimits { + metadata.addString(ulimit, forKey: "ulimit") + } + if let hostname = config.hostname { + metadata.addString(hostname, forKey: "hostname") + } + if let shmSize = config.shmSize { + metadata.addString(String(shmSize), forKey: "shm-size") + } + if let cgroupParent = config.cgroupParent { + metadata.addString(cgroupParent, forKey: "cgroup-parent") + } + if let network = config.network { + metadata.addString(network, forKey: "network") + } for (id, data) in config.secrets { metadata.addString(id + "=" + data.base64EncodedString(), forKey: "secrets") } diff --git a/Sources/ContainerCommands/BuildCommand.swift b/Sources/ContainerCommands/BuildCommand.swift index c216bc26b..9c1cbd572 100644 --- a/Sources/ContainerCommands/BuildCommand.swift +++ b/Sources/ContainerCommands/BuildCommand.swift @@ -22,6 +22,7 @@ import ContainerPersistence import ContainerPlugin import Containerization import ContainerizationError +import ContainerizationExtras import ContainerizationOCI import ContainerizationOS import Foundation @@ -63,6 +64,27 @@ extension Application { @Option(name: .long, help: ArgumentHelp("Set build-time variables", valueName: "key=val")) var buildArg: [String] = [] + @Option(name: .long, help: ArgumentHelp("Set build-contexts. Relative Paths are resolved based on the current working directory.", valueName: "name=")) + var buildContext: [String] = [] + + @Option(name: .long, help: ArgumentHelp("Add a host-to-IP mapping resolvable during the build", valueName: "host=ip")) + var addHost: [String] = [] + + @Option(name: .long, help: ArgumentHelp("Hostname the build environment reports", valueName: "name")) + var hostname: String? + + @Option(name: .long, help: ArgumentHelp("Size of /dev/shm during the build", valueName: "bytes")) + var shmSize: String? + + @Option(name: .long, help: ArgumentHelp("Set a resource limit for the build", valueName: "type=soft:hard")) + var ulimit: [String] = [] + + @Option(name: .long, help: ArgumentHelp("Parent cgroup for the build environment", valueName: "cgroup")) + var cgroupParent: String? + + @Option(name: .long, help: ArgumentHelp("Network mode for the build (none, host or sandbox)", valueName: "mode")) + var network: String? + @Option(name: .long, help: ArgumentHelp("Cache imports for the build", valueName: "value", visibility: .hidden)) var cacheIn: [String] = { [] @@ -81,6 +103,10 @@ extension Application { var dockerfile: String = "-" + /// `--shm-size` parsed to bytes. The frontend takes a plain byte count, + /// so the suffixed forms users expect from docker are resolved here. + var shmSizeBytes: UInt64? + @Option(name: .shortAndLong, help: ArgumentHelp("Set a label", valueName: "key=val")) var label: [String] = [] @@ -370,13 +396,21 @@ extension Application { }() group.addTask { [ - terminal, buildArg, secretsData, ssh, contextDir, ignoreFileData, label, noCache, target, quiet, cacheIn, cacheOut, pull, exports, imageNames, tempURL, - log + terminal, buildArg, buildContext, addHost, hostname, shmSizeBytes, ulimit, cgroupParent, network, + secretsData, ssh, contextDir, ignoreFileData, label, noCache, target, quiet, cacheIn, cacheOut, pull, exports, + imageNames, tempURL, log, ] in let config = Builder.BuildConfig( buildID: buildID, contentStore: RemoteContentStoreClient(), buildArgs: buildArg, + buildContexts: buildContext, + addHosts: addHost, + ulimits: ulimit, + hostname: hostname, + shmSize: shmSizeBytes, + cgroupParent: cgroupParent, + network: network, secrets: secretsData, ssh: ssh, contextDir: contextDir, @@ -483,6 +517,88 @@ extension Application { } } + // Each --build-context is name=value, where the value is either a + // non-local reference (image, git, URL, oci-layout) passed through + // to the builder, or a local directory. A local directory must + // exist, and is resolved to an absolute path here so that neither + // the API server's nor the builder's working directory can change + // its meaning. + let passthroughPrefixes = [ + "docker-image://", "oci-layout:", "http://", "https://", + "git://", "git@", "ssh://", "local:", "input:", + ] + buildContext = try buildContext.map { entry in + let parts = entry.split(separator: "=", maxSplits: 1) + guard parts.count == 2, !parts[0].isEmpty, !parts[1].isEmpty else { + throw ValidationError("build context must be name=value: \(entry)") + } + let value = String(parts[1]) + if passthroughPrefixes.contains(where: { value.hasPrefix($0) }) { + // An oci-layout value names a layout directory on this + // machine, so a missing directory can be refused here with + // its real cause; anything that slips past resolves as an + // image reference and fails with a misleading + // invalid-domain error. + if value.hasPrefix("oci-layout://") { + var path = String(value.dropFirst("oci-layout://".count)) + if let at = path.firstIndex(of: "@") { + path = String(path[path.startIndex.. BuildTransfer { + var p = readPacket(source: source) + p.metadata["dir-name"] = dirName + return p + } + + @Test func testInfoServesFromNamedContextRoot() async throws { + let namedDir = base.appendingPathComponent("dep") + try fm.createDirectory(at: namedDir, withIntermediateDirectories: true) + try write("hello", to: namedDir.appendingPathComponent("hello.txt")) + + let fssync = try BuildFSSync(contextDir, namedContexts: ["dep": namedDir]) + var continuation: AsyncStream.Continuation! + _ = AsyncStream { continuation = $0 } + defer { continuation.finish() } + + // hello.txt exists only under the named root; resolving it proves the + // dir-name selected that root rather than the primary context. + try await fssync.info(continuation, readPacket(source: "hello.txt", dirName: "dep"), "build-0") + } + + @Test func testInfoPrimaryNamesResolveToContextDir() async throws { + let namedDir = base.appendingPathComponent("dep") + try fm.createDirectory(at: namedDir, withIntermediateDirectories: true) + try write("primary", to: contextDir.appendingPathComponent("main.txt")) + + let fssync = try BuildFSSync(contextDir, namedContexts: ["dep": namedDir]) + var continuation: AsyncStream.Continuation! + _ = AsyncStream { continuation = $0 } + defer { continuation.finish() } + + // main.txt exists only in the primary context; both reserved names and + // the no-metadata packet must resolve there even with named contexts + // declared. + try await fssync.info(continuation, readPacket(source: "main.txt", dirName: "context"), "build-0") + try await fssync.info(continuation, readPacket(source: "main.txt", dirName: "dockerfile"), "build-0") + try await fssync.info(continuation, readPacket(source: "main.txt"), "build-0") + } + + @Test func testReadRejectsUndeclaredNamedContext() async throws { + try write("primary", to: contextDir.appendingPathComponent("main.txt")) + + let fssync = try BuildFSSync(contextDir) + var continuation: AsyncStream.Continuation! + _ = AsyncStream { continuation = $0 } + defer { continuation.finish() } + + do { + try await fssync.read(continuation, readPacket(source: "main.txt", dirName: "undeclared"), "build-0") + Issue.record("read() should refuse a dir-name that was never declared on the command line") + } catch let error as BuildFSSync.Error { + #expect(error == .unknownNamedContext("undeclared")) + } + } + + @Test func testReadRejectsSymlinkEscapeFromNamedContextRoot() async throws { + let namedDir = base.appendingPathComponent("dep") + try fm.createDirectory(at: namedDir, withIntermediateDirectories: true) + let secretFile = outsideDir.appendingPathComponent("secret.txt") + try write("supersecret", to: secretFile) + + // Symlink inside the named root → absolute path outside it. The + // primary-context tests above prove this is refused under contextDir; + // this proves the same enforcement holds under a named root. + try fm.createSymbolicLink( + atPath: namedDir.appendingPathComponent("leak").path(percentEncoded: false), + withDestinationPath: secretFile.path(percentEncoded: false) + ) + + let fssync = try BuildFSSync(contextDir, namedContexts: ["dep": namedDir]) + var continuation: AsyncStream.Continuation! + _ = AsyncStream { continuation = $0 } + defer { continuation.finish() } + + do { + try await fssync.read(continuation, readPacket(source: "leak", dirName: "dep"), "build-0") + Issue.record("read() should throw BuildFSSync.Error for a symlink that resolves outside its named context root") + } catch is BuildFSSync.Error { + // expected + } + } + + @Test func testInitRejectsMissingNamedContextDirectory() async throws { + let missing = base.appendingPathComponent("never-created") + do { + _ = try BuildFSSync(contextDir, namedContexts: ["dep": missing]) + Issue.record("init should refuse a named context directory that does not exist") + } catch is BuildFSSync.Error { + // expected + } + } }