diff --git a/Sources/ContainerBuild/BuildAPI+Extensions.swift b/Sources/ContainerBuild/BuildAPI+Extensions.swift index b052b1042..c91416d6a 100644 --- a/Sources/ContainerBuild/BuildAPI+Extensions.swift +++ b/Sources/ContainerBuild/BuildAPI+Extensions.swift @@ -36,6 +36,14 @@ extension BuildTransfer { return method == "" ? nil : method } + /// The BuildKit local-dir name this transfer belongs to: empty or + /// "context" for the primary build context, otherwise the name of a + /// `--build-context` the CLI declared. + func dirName() -> String? { + let dirName = self.metadata["dir-name"] + return dirName == "" ? nil : dirName + } + func includePatterns() -> [String]? { guard let includePatternsString = self.metadata["include-patterns"] else { return nil diff --git a/Sources/ContainerBuild/BuildFSSync.swift b/Sources/ContainerBuild/BuildFSSync.swift index c5a5288f2..27dccece7 100644 --- a/Sources/ContainerBuild/BuildFSSync.swift +++ b/Sources/ContainerBuild/BuildFSSync.swift @@ -55,7 +55,13 @@ import GRPCCore actor BuildFSSync: BuildPipelineHandler { let contextDir: URL - init(_ contextDir: URL) throws { + /// Local `--build-context` directories by name. The host is the sole + /// authority for this mapping: the shim only ever names a context, and a + /// name outside this map is refused, so the builder VM cannot steer the + /// host toward a path that was never declared on the command line. + let namedContexts: [String: URL] + + init(_ contextDir: URL, namedContexts: [String: URL] = [:]) throws { let resolved = contextDir.resolvingSymlinksInPath() guard FileManager.default.fileExists(atPath: contextDir.cleanPath) else { throw Error.contextNotFound(contextDir.cleanPath) @@ -64,7 +70,42 @@ actor BuildFSSync: BuildPipelineHandler { throw Error.contextIsNotDirectory(contextDir.cleanPath) } + var resolvedNamed: [String: URL] = [:] + for (name, dir) in namedContexts { + let resolvedDir = dir.resolvingSymlinksInPath() + guard FileManager.default.fileExists(atPath: dir.cleanPath) else { + throw Error.contextNotFound(dir.cleanPath) + } + guard resolvedDir.isDirectory else { + throw Error.contextIsNotDirectory(dir.cleanPath) + } + resolvedNamed[name] = resolvedDir + } + self.contextDir = resolved + self.namedContexts = resolvedNamed + } + + /// Resolve the context root a transfer operates on from the dir-name the + /// shim relayed. Every serving path (walk, read, info) resolves through + /// here, so the same containment enforcement applies to every root. + /// + /// This mirrors BuildKit's own session provider, which resolves the + /// `dir-name` metadata through a registered directory source and answers + /// NotFound for a name it does not hold, so a name is the only thing a + /// requester can choose and the server keeps the paths. The primary + /// context and the dockerfile directory arrive under BuildKit's default + /// names for them. + /// https://github.com/moby/buildkit/blob/v0.29.0/session/filesync/filesync.go + /// https://github.com/moby/buildkit/blob/v0.29.0/frontend/dockerui/context.go + private func contextRoot(_ packet: BuildTransfer) throws -> URL { + guard let name = packet.dirName(), name != "context", name != "dockerfile" else { + return self.contextDir + } + guard let root = self.namedContexts[name] else { + throw Error.unknownNamedContext(name) + } + return root } nonisolated func accept(_ packet: ServerStream) throws -> Bool { @@ -102,22 +143,23 @@ actor BuildFSSync: BuildPipelineHandler { func read(_ sender: AsyncStream.Continuation, _ packet: BuildTransfer, _ buildID: String) async throws { let offset: UInt64 = packet.offset() ?? 0 let size: Int = packet.len() ?? 0 + let root = try contextRoot(packet) var path: URL if packet.source.hasPrefix("/") { path = URL(fileURLWithPath: packet.source).standardizedFileURL } else { path = - contextDir + root .appendingPathComponent(packet.source) .standardizedFileURL } if !FileManager.default.fileExists(atPath: path.cleanPath) { - path = URL(filePath: self.contextDir.cleanPath) + path = URL(filePath: root.cleanPath) path.append(components: packet.source.cleanPathComponent) } let resolved = path.resolvingSymlinksInPath() - guard self.contextDir.parentOf(resolved) else { - throw Error.pathIsNotChild(resolved.cleanPath, self.contextDir.cleanPath) + guard root.parentOf(resolved) else { + throw Error.pathIsNotChild(resolved.cleanPath, root.cleanPath) } let data = try { if try path.isDir() { @@ -127,7 +169,7 @@ actor BuildFSSync: BuildPipelineHandler { return try file.data(offset: offset, length: size) ?? Data() }() - let transfer = try path.buildTransfer(id: packet.id, contextDir: self.contextDir, complete: true, data: data) + let transfer = try path.buildTransfer(id: packet.id, contextDir: root, complete: true, data: data) var response = ClientStream() response.buildID = buildID response.buildTransfer = transfer @@ -142,20 +184,21 @@ actor BuildFSSync: BuildPipelineHandler { /// normal `Walk`-based build. Must reject paths that escape the context root /// via symlinks for the same reasons as ``read(_:_:_:)``. func info(_ sender: AsyncStream.Continuation, _ packet: BuildTransfer, _ buildID: String) async throws { + let root = try contextRoot(packet) let path: URL if packet.source.hasPrefix("/") { path = URL(fileURLWithPath: packet.source).standardizedFileURL } else { path = - contextDir + root .appendingPathComponent(packet.source) .standardizedFileURL } let resolved = path.resolvingSymlinksInPath() - guard self.contextDir.parentOf(resolved) else { - throw Error.pathIsNotChild(resolved.cleanPath, self.contextDir.cleanPath) + guard root.parentOf(resolved) else { + throw Error.pathIsNotChild(resolved.cleanPath, root.cleanPath) } - let transfer = try path.buildTransfer(id: packet.id, contextDir: self.contextDir, complete: true) + let transfer = try path.buildTransfer(id: packet.id, contextDir: root, complete: true) var response = ClientStream() response.buildID = buildID response.buildTransfer = transfer @@ -200,30 +243,32 @@ actor BuildFSSync: BuildPipelineHandler { _ buildID: String ) async throws { let wantsTar = packet.mode() == "tar" + let root = try contextRoot(packet) var entries: [String: Set] = [:] let followPaths: [String] = packet.followPaths() ?? [] - let followPathsWalked = try walk(root: self.contextDir, includePatterns: followPaths) + let followPathsWalked = try walk(root: root, includePatterns: followPaths) + for url in followPathsWalked { - guard self.contextDir.absoluteURL.cleanPath != url.absoluteURL.cleanPath else { + guard root.absoluteURL.cleanPath != url.absoluteURL.cleanPath else { continue } - guard self.contextDir.parentOf(url) else { + guard root.parentOf(url) else { continue } - let relPath = try url.relativeChildPath(to: contextDir) - let parentPath = try url.deletingLastPathComponent().relativeChildPath(to: contextDir) + let relPath = try url.relativeChildPath(to: root) + let parentPath = try url.deletingLastPathComponent().relativeChildPath(to: root) let entry = DirEntry(url: url, isDirectory: url.hasDirectoryPath, relativePath: relPath) entries[parentPath, default: []].insert(entry) if url.isSymlink { let target: URL = url.resolvingSymlinksInPath() - if self.contextDir.parentOf(target) { - let relPath = try target.relativeChildPath(to: self.contextDir) + if root.parentOf(target) { + let relPath = try target.relativeChildPath(to: root) let entry = DirEntry(url: target, isDirectory: target.hasDirectoryPath, relativePath: relPath) - let parentPath: String = try target.deletingLastPathComponent().relativeChildPath(to: self.contextDir) + let parentPath: String = try target.deletingLastPathComponent().relativeChildPath(to: root) entries[parentPath, default: []].insert(entry) } } @@ -234,7 +279,7 @@ actor BuildFSSync: BuildPipelineHandler { if !wantsTar { let fileInfos = try fileOrder.map { rel -> FileInfo in - try FileInfo(path: contextDir.appendingPathComponent(rel), contextDir: contextDir) + try FileInfo(path: root.appendingPathComponent(rel), contextDir: root) } let data = try JSONEncoder().encode(fileInfos) @@ -268,15 +313,15 @@ actor BuildFSSync: BuildPipelineHandler { filter: .none) let tarHash = try Archiver.compress( - source: contextDir, + source: root, destination: tarURL, writerConfiguration: writerCfg ) { url in - guard let rel = try? url.relativeChildPath(to: contextDir) else { + guard let rel = try? url.relativeChildPath(to: root) else { return nil } - guard let parent = try? url.deletingLastPathComponent().relativeChildPath(to: self.contextDir) else { + guard let parent = try? url.deletingLastPathComponent().relativeChildPath(to: root) else { return nil } @@ -450,6 +495,7 @@ extension BuildFSSync { case couldNotDetermineUID(String) case couldNotDetermineGID(String) case pathIsNotChild(String, String) + case unknownNamedContext(String) var description: String { switch self { @@ -477,6 +523,8 @@ extension BuildFSSync { return "could not determine GID of file at path: \(path)" case .pathIsNotChild(let path, let parent): return "\(path) is not a child of \(parent)" + case .unknownNamedContext(let name): + return "no --build-context named \(name) was declared" } } } diff --git a/Sources/ContainerBuild/BuildImageResolver.swift b/Sources/ContainerBuild/BuildImageResolver.swift index 5b15352d8..9768aba3a 100644 --- a/Sources/ContainerBuild/BuildImageResolver.swift +++ b/Sources/ContainerBuild/BuildImageResolver.swift @@ -17,6 +17,7 @@ import ContainerAPIClient import ContainerPersistence import Containerization +import ContainerizationArchive import ContainerizationOCI import Foundation import GRPCCore @@ -85,6 +86,12 @@ struct BuildImageResolver: BuildPipelineHandler { let progress = ProgressBar(config: progressConfig) defer { progress.finish() } progress.start() + if let parsed = parseLocalRef(ref: ref) { + guard let image = try await handleLocalUCI(url: parsed.url, digest: parsed.digest) else { + throw Error.imageNotFound + } + return image + } if self.pull { return try await ClientImage.pull(reference: ref, platform: platform, containerSystemConfig: containerSystemConfig, progressUpdate: progress.handler) @@ -121,6 +128,79 @@ struct BuildImageResolver: BuildPipelineHandler { } throw Error.unknownPlatformForImage(platform.description, ref) } + + // handle oci-layout from build-context + // Not throwing here so that we can fall back to pull or fetch + private func parseLocalRef(ref: String) -> (url: URL, digest: String?)? { + guard let range = ref.firstRange(of: "oci-layout://") else { + return nil + } + var ref = ref + ref.removeSubrange(range) + + let digest = extractDigest(ref) + if let digest, let range = ref.range(of: digest) { + ref.removeSubrange(range) + } + ref = ref.trimmingCharacters(in: ["@"]) + if !FileManager.default.fileExists(atPath: ref) { + return nil + } + let url = URL(filePath: ref) + guard url.isDirectory else { + return nil + } + return (url, digest) + } + + private func handleLocalUCI(url: URL, digest: String?) async throws -> ClientImage? { + let tarURL = URL.temporaryDirectory + .appendingPathComponent(UUID().uuidString + ".tar") + + defer { try? FileManager.default.removeItem(at: tarURL) } + + let writerCfg = ArchiveWriterConfiguration( + format: .paxRestricted, + filter: .none + ) + + let root = url + + let _ = try Archiver.compress( + source: url, + destination: tarURL, + writerConfiguration: writerCfg, + closure: { url in + guard let rel = try? url.relativeChildPath(to: root) else { + return nil + } + return Archiver.ArchiveEntryInfo( + pathOnHost: url, + pathInArchive: URL(fileURLWithPath: rel) + ) + } + ) + + let result = try await ClientImage.load(from: tarURL.absolutePath(), force: true) + let images = result.images + let first = images.first + if let digest { + return images.first(where: { $0.digest == digest }) ?? first + } + return first + } + + private func extractDigest(_ string: String) -> String? { + guard let reg = try? Regex("[A-Za-z][A-Za-z0-9]*(?:[-_+.][A-Za-z][A-Za-z0-9]*)*[:][[:xdigit:]]{32,}") else { + return nil + } + let matches = string.matches(of: reg) + // digest always comes last when specified + guard let last = matches.last else { + return nil + } + return String(string[last.range]) + } } extension ImageTransfer { diff --git a/Sources/ContainerBuild/BuildPipelineHandler.swift b/Sources/ContainerBuild/BuildPipelineHandler.swift index da0b0081e..d0fd11367 100644 --- a/Sources/ContainerBuild/BuildPipelineHandler.swift +++ b/Sources/ContainerBuild/BuildPipelineHandler.swift @@ -75,11 +75,28 @@ protocol BuildPipelineHandler: Sendable { /// ``` public actor BuildPipeline { let handlers: [BuildPipelineHandler] + /// The handler holding what the build has gathered, kept by its own type + /// so the build can drop it when it ends. + private let content: BuildRemoteContentProxy public init(_ config: Builder.BuildConfig) async throws { + // Local named contexts are the `name=/absolute/path` entries; the CLI + // resolved every local value to an absolute path when it validated the + // flag, so a leading slash is what distinguishes a directory from an + // image, git, URL or oci-layout reference here. + var namedContexts: [String: URL] = [:] + for entry in config.buildContexts { + let parts = entry.split(separator: "=", maxSplits: 1) + guard parts.count == 2, parts[1].hasPrefix("/") else { + continue + } + namedContexts[String(parts[0])] = URL(filePath: String(parts[1])) + } + let content = try BuildRemoteContentProxy(config.contentStore, ingestDir: config.ingestDir) + self.content = content self.handlers = [ - try BuildFSSync(URL(filePath: config.contextDir)), - try BuildRemoteContentProxy(config.contentStore), + try BuildFSSync(URL(filePath: config.contextDir), namedContexts: namedContexts), + content, try BuildImageResolver( config.contentStore, quiet: config.quiet, @@ -112,6 +129,15 @@ public actor BuildPipeline { } } + /// Lets go of the blobs the build began and never sealed. + /// + /// The build owns this the way it owns the ingest session it cancels when + /// it fails: what was gathered for an image that will not be recorded is + /// the build's to drop, and the build's end is when that is known. + public func discardGathered() async { + await self.content.writes.discardAll() + } + /// untilFirstError() throws when any one of its submitted tasks fail. /// This is useful for asynchronous packet processing scenarios which /// have the following 3 requirements: diff --git a/Sources/ContainerBuild/BuildRemoteContentProxy.swift b/Sources/ContainerBuild/BuildRemoteContentProxy.swift index afc1e2707..209ece13a 100644 --- a/Sources/ContainerBuild/BuildRemoteContentProxy.swift +++ b/Sources/ContainerBuild/BuildRemoteContentProxy.swift @@ -17,7 +17,9 @@ import ContainerAPIClient import Containerization import ContainerizationArchive +import ContainerizationError import ContainerizationOCI +import CryptoKit import Foundation import GRPCCore @@ -28,9 +30,152 @@ import GRPCCore /// base-image layers that are not already present in the builder VM. struct BuildRemoteContentProxy: BuildPipelineHandler { let local: ContentStore + let writes: WriteSessions + /// Where a committed blob is put. The store gains the whole set when the + /// build's images are recorded, so until then a blob lives here and the + /// store does not hold it. Reads answer from here as well, because a + /// writer asks the store what it holds before writing and reads back what + /// it wrote, and neither can see this directory. + let ingestDir: URL? - public init(_ contentStore: ContentStore) throws { + public init(_ contentStore: ContentStore, ingestDir: URL? = nil) throws { self.local = contentStore + self.writes = WriteSessions() + self.ingestDir = ingestDir + } + + /// The path a committed blob takes in the gathering directory, which names + /// it by digest alone under the store's algorithm directory, the way the + /// store names it. + private func gathered(_ digest: String) -> URL? { + self.ingestDir?.appendingPathComponent(digest.trimmingDigestPrefix) + } + + /// A blob the build holds, whether the store has taken it or it is still + /// gathered here. Asking the store first and the gathering directory + /// second is how importing an image resolves a blob it may itself have + /// only just written. + private func held(_ digest: String) async throws -> Content? { + if let content = try await local.get(digest: digest) { + return content + } + guard let path = gathered(digest) else { + return nil + } + return try? LocalContent(path: path) + } + + /// The same, for the question a writer asks before writing. + /// + /// A writer skips whatever the store says it holds, so a blob answered for + /// here is one this build will not write and still needs: it belongs to + /// the image about to be recorded, and until that record exists nothing + /// claims it and a sweep is free to take it. The build gathers a copy, so + /// what it names is what it hands over. Importing an image copies the + /// blobs the store already holds into its own ingest for the same reason. + private func heldForWriter(_ digest: String) async throws -> Content? { + guard let path = gathered(digest) else { + return try await local.get(digest: digest) + } + if let content = try? LocalContent(path: path) { + return content + } + guard let found = try await local.get(digest: digest) else { + return nil + } + try? FileManager.default.copyItem(at: found.path, to: path) + return found + } + + /// The blobs BuildKit is streaming into the store, one temporary file per + /// ref, appended chunk by chunk until the commit seals it. The digest is + /// computed as the chunks arrive, the way containerd's own ingest writer + /// keeps it, so the commit finalizes and compares rather than re-reading + /// the file. + actor WriteSessions { + struct Open { + let url: URL + let handle: FileHandle + var written: UInt64 + var hasher: SHA256 + } + + private var open: [String: Open] = [:] + /// Where this build's blobs are written, made when the first one + /// arrives and taken away when the build ends. + let dir: URL + + init() { + self.dir = FileManager.default.temporaryDirectory + .appendingPathComponent("build-blob-writes-\(UUID().uuidString)") + } + + func append(ref: String, offset: UInt64, data: Data) throws -> UInt64 { + var session: Open + if let existing = open[ref] { + session = existing + } else { + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + let url = dir.appendingPathComponent(UUID().uuidString) + FileManager.default.createFile(atPath: url.path, contents: nil) + session = Open(url: url, handle: try FileHandle(forWritingTo: url), written: 0, hasher: SHA256()) + } + // An offset the write names must be the one the session is at, + // and a zero offset asks for the blob again from its beginning: + // the session drops the bytes it holds and rebuilds the digest + // over what arrives next, which is how a writer restarts a blob + // it has already begun. + // https://github.com/containerd/containerd/blob/main/plugins/services/content/contentserver/contentserver.go + // https://github.com/containerd/containerd/blob/main/plugins/content/local/writer.go + if offset > 0 { + guard offset == session.written else { + throw ContainerizationError( + .invalidArgument, + message: "write for \(ref) at offset \(offset), \(session.written) bytes held" + ) + } + } else if session.written > 0 { + session.written = 0 + session.hasher = SHA256() + try session.handle.seek(toOffset: 0) + try session.handle.truncate(atOffset: 0) + } + try session.handle.write(contentsOf: data) + session.written += UInt64(data.count) + session.hasher.update(data: data) + open[ref] = session + return session.written + } + + /// Lets go of every blob still being written and takes the directory + /// holding them with it. + /// + /// A blob is committed one at a time, and the commit takes its file out + /// of here; what remains when the build ends is what the build began + /// and never sealed, which nothing will ask for again. The protocol + /// carries no word for abandoning a single blob, so the end of the + /// stream is the only word there is. + func discardAll() { + for session in open.values { + try? session.handle.close() + try? FileManager.default.removeItem(at: session.url) + } + open.removeAll() + try? FileManager.default.removeItem(at: dir) + } + + func close(ref: String) throws -> (url: URL, written: UInt64, digest: String)? { + guard let session = open.removeValue(forKey: ref) else { + return nil + } + // The bytes must be on disk before the commit renames the file + // into the store, or a crash commits a hole where content should + // be, which is why the reference syncs before it stats. + try session.handle.synchronize() + try session.handle.close() + let digest = "sha256:" + session.hasher.finalize().map { String(format: "%02x", $0) }.joined() + return (session.url, session.written, digest) + } } func accept(_ packet: ServerStream) throws -> Bool { @@ -57,20 +202,31 @@ struct BuildRemoteContentProxy: BuildPipelineHandler { try await self.info(sender, imageTransfer, packet.buildID) case .readerAt: try await self.readerAt(sender, imageTransfer, packet.buildID) + case .write: + try await self.write(sender, imageTransfer, packet.buildID) default: throw Error.unknownMethod(method) } } func info(_ sender: AsyncStream.Continuation, _ packet: ImageTransfer, _ buildID: String) async throws { - let descriptor = try await local.get(digest: packet.tag) - let size = try descriptor?.size() - let transfer = try ImageTransfer( + let content = try await heldForWriter(packet.tag) + let size = try content?.size() + var transfer = try ImageTransfer( id: packet.id, digest: packet.tag, method: ContentStoreMethod.info.rawValue, size: size ) + if content == nil { + // A store answers an info request for a digest it does not hold + // with not found, and that answer is what tells a writer the blob + // has still to be written: an export asks before it writes and + // skips whatever the store says it already holds, so an answer + // describing a blob that is not there loses the export's blobs. + // https://github.com/containerd/containerd/blob/main/plugins/content/local/store.go + transfer.metadata["error"] = "content \(packet.tag) not found" + } var response = ClientStream() response.buildID = buildID response.imageTransfer = transfer @@ -82,7 +238,7 @@ struct BuildRemoteContentProxy: BuildPipelineHandler { let digest = packet.descriptor.digest let offset: UInt64 = packet.offset() ?? 0 let size: Int = packet.len() ?? 0 - guard let descriptor = try await local.get(digest: digest) else { + guard let content = try await held(digest) else { throw Error.contentMissing } if offset == 0 && size == 0 { // Metadata request @@ -90,7 +246,7 @@ struct BuildRemoteContentProxy: BuildPipelineHandler { id: packet.id, digest: packet.tag, method: ContentStoreMethod.readerAt.rawValue, - size: descriptor.size(), + size: content.size(), data: Data() ) transfer.complete = true @@ -101,7 +257,7 @@ struct BuildRemoteContentProxy: BuildPipelineHandler { sender.yield(response) return } - guard let data = try descriptor.data(offset: offset, length: size) else { + guard let data = try content.data(offset: offset, length: size) else { throw Error.invalidOffsetSizeForContent(packet.descriptor.digest, offset, size) } @@ -119,6 +275,112 @@ struct BuildRemoteContentProxy: BuildPipelineHandler { sender.yield(response) } + /// A blob BuildKit exports arrives as write packets carrying sequential + /// chunks and a commit packet carrying the digest and size the whole must + /// verify to. The chunks gather in a temporary file, and the commit moves + /// it into the store through an ingest session, which is what dedups a + /// blob the store already holds. The commit's checks are the receiving + /// store's obligations under containerd's writer contract, kept in the + /// reference's order and spellings: + /// https://github.com/containerd/containerd/blob/main/plugins/content/local/writer.go + /// The reply mirrors the shim's writer contract: offset for a write, + /// exists for a commit the store already satisfied, error for anything + /// that failed. + func write(_ sender: AsyncStream.Continuation, _ packet: ImageTransfer, _ buildID: String) async throws { + let action = packet.metadata["action"] ?? "" + let ref = packet.metadata["ref"] ?? packet.id + var reply: [String: String] = [:] + + do { + switch action { + case "write": + let offset = UInt64(packet.metadata["offset"] ?? "0") ?? 0 + let written = try await writes.append(ref: ref, offset: offset, data: packet.data) + reply["offset"] = String(written) + case "commit": + guard let expected = packet.metadata["expected"] else { + throw ContainerizationError(.invalidArgument, message: "commit for \(ref) names no digest") + } + let total = UInt64(packet.metadata["total"] ?? "0") ?? 0 + let closed = try await writes.close(ref: ref) + do { + if try await local.get(digest: expected) != nil { + reply["exists"] = "true" + if let closed { + try? FileManager.default.removeItem(at: closed.url) + } + break + } + guard let closed else { + throw ContainerizationError(.invalidArgument, message: "commit for \(ref) with no bytes written") + } + guard total == 0 || closed.written == total else { + throw ContainerizationError( + .invalidArgument, + message: "unexpected commit size \(closed.written), expected \(total) for \(ref)" + ) + } + guard closed.digest == expected else { + throw ContainerizationError( + .invalidArgument, + message: "unexpected commit digest \(closed.digest), expected \(expected) for \(ref)" + ) + } + // A blob is named for its digest alone under the store's + // algorithm directory, which is the name a later read + // resolves; a name carrying the algorithm again lands a + // file nothing looks for. + // + // The blob is gathered rather than handed to the store, + // and the store takes the whole set when it records the + // images naming them, so it never holds a blob that no + // image claims. A build given nowhere to gather hands it + // over as it arrives. + if let destination = gathered(expected) { + if FileManager.default.fileExists(atPath: destination.path) { + try FileManager.default.removeItem(at: closed.url) + } else { + try FileManager.default.moveItem(at: closed.url, to: destination) + } + } else { + let session = try await local.newIngestSession() + try FileManager.default.moveItem( + at: closed.url, + to: session.ingestDir.appendingPathComponent(expected.trimmingDigestPrefix) + ) + _ = try await local.completeIngestSession(session.id) + } + reply["offset"] = String(closed.written) + } catch { + // A failed commit leaves nothing behind, the way the + // reference removes its ingest path on the way out. + if let closed { + try? FileManager.default.removeItem(at: closed.url) + } + throw error + } + default: + throw Error.unknownMethod("\(ContentStoreMethod.write.rawValue) action \(action)") + } + } catch { + reply["error"] = "\(error)" + } + + var transfer = try ImageTransfer( + id: packet.id, + digest: packet.tag, + method: ContentStoreMethod.write.rawValue + ) + for (key, value) in reply { + transfer.metadata[key] = value + } + var response = ClientStream() + response.buildID = buildID + response.imageTransfer = transfer + response.packetType = .imageTransfer(transfer) + sender.yield(response) + } + func delete(_ sender: AsyncStream.Continuation, _ packet: ImageTransfer) async throws { throw NSError(domain: "RemoteContentProxy", code: 1, userInfo: [NSLocalizedDescriptionKey: "unimplemented method \(ContentStoreMethod.delete)"]) } @@ -134,6 +396,7 @@ struct BuildRemoteContentProxy: BuildPipelineHandler { enum ContentStoreMethod: String { case info = "/containerd.services.content.v1.Content/Info" case readerAt = "/containerd.services.content.v1.Content/ReaderAt" + case write = "/containerd.services.content.v1.Content/Write" case delete = "/containerd.services.content.v1.Content/Delete" case update = "/containerd.services.content.v1.Content/Update" case walk = "/containerd.services.content.v1.Content/Walk" diff --git a/Sources/ContainerBuild/Builder.swift b/Sources/ContainerBuild/Builder.swift index 151ee033e..9c3464081 100644 --- a/Sources/ContainerBuild/Builder.swift +++ b/Sources/ContainerBuild/Builder.swift @@ -151,10 +151,14 @@ public struct Builder: Sendable { } ) } catch Error.buildComplete { + await pipeline.discardGathered() self.grpcClient.beginGracefulShutdown() self.clientTask.cancel() try await group.shutdownGracefully() return + } catch { + await pipeline.discardGathered() + throw error } } @@ -199,7 +203,7 @@ public struct Builder: Sendable { } switch type { - case "oci": + case "oci", "store": break case "tar": if destinationValue == nil { @@ -267,6 +271,13 @@ public struct Builder: Sendable { public let buildID: String public let contentStore: ContentStore public let buildArgs: [String] + public let buildContexts: [String] + public let addHosts: [String] + public let ulimits: [String] + public let hostname: String? + public let shmSize: UInt64? + public let cgroupParent: String? + public let network: String? public let secrets: [String: Data] public let ssh: String public let contextDir: String @@ -284,11 +295,24 @@ public struct Builder: Sendable { public let cacheOut: [String] public let pull: Bool public let containerSystemConfig: ContainerSystemConfig + /// Where the blobs this build exports are gathered before the store + /// takes them. They land there rather than in the store itself so + /// that the store gains them at the moment it records the image that + /// names them, and never holds one that nothing claims. + public let ingestDir: URL? public init( buildID: String, contentStore: ContentStore, + ingestDir: URL? = nil, buildArgs: [String], + buildContexts: [String], + addHosts: [String] = [], + ulimits: [String] = [], + hostname: String? = nil, + shmSize: UInt64? = nil, + cgroupParent: String? = nil, + network: String? = nil, secrets: [String: Data], ssh: String, contextDir: String, @@ -309,7 +333,15 @@ public struct Builder: Sendable { ) { self.buildID = buildID self.contentStore = contentStore + self.ingestDir = ingestDir self.buildArgs = buildArgs + self.buildContexts = buildContexts + self.addHosts = addHosts + self.ulimits = ulimits + self.hostname = hostname + self.shmSize = shmSize + self.cgroupParent = cgroupParent + self.network = network self.secrets = secrets self.ssh = ssh self.contextDir = contextDir @@ -356,6 +388,27 @@ public struct Builder: Sendable { for buildArg in config.buildArgs { metadata.addString(buildArg, forKey: "build-args") } + for buildContext in config.buildContexts { + metadata.addString(buildContext, forKey: "build-context") + } + for addHost in config.addHosts { + metadata.addString(addHost, forKey: "add-host") + } + for ulimit in config.ulimits { + metadata.addString(ulimit, forKey: "ulimit") + } + if let hostname = config.hostname { + metadata.addString(hostname, forKey: "hostname") + } + if let shmSize = config.shmSize { + metadata.addString(String(shmSize), forKey: "shm-size") + } + if let cgroupParent = config.cgroupParent { + metadata.addString(cgroupParent, forKey: "cgroup-parent") + } + if let network = config.network { + metadata.addString(network, forKey: "network") + } for (id, data) in config.secrets { metadata.addString(id + "=" + data.base64EncodedString(), forKey: "secrets") } @@ -363,6 +416,12 @@ public struct Builder: Sendable { metadata.addString("default", forKey: "ssh") } for output in config.exports { + // A store export is the builder's default when it is told nothing: + // blobs land in the content store through the session and only the + // root digest comes back, so the entry sends no output string. + if output.type == "store" { + continue + } metadata.addString(try output.stringValue, forKey: "outputs") } for cacheIn in config.cacheIn { diff --git a/Sources/ContainerCommands/BuildCommand.swift b/Sources/ContainerCommands/BuildCommand.swift index c216bc26b..67f5b738c 100644 --- a/Sources/ContainerCommands/BuildCommand.swift +++ b/Sources/ContainerCommands/BuildCommand.swift @@ -22,6 +22,7 @@ import ContainerPersistence import ContainerPlugin import Containerization import ContainerizationError +import ContainerizationExtras import ContainerizationOCI import ContainerizationOS import Foundation @@ -63,6 +64,27 @@ extension Application { @Option(name: .long, help: ArgumentHelp("Set build-time variables", valueName: "key=val")) var buildArg: [String] = [] + @Option(name: .long, help: ArgumentHelp("Set build-contexts. Relative Paths are resolved based on the current working directory.", valueName: "name=")) + var buildContext: [String] = [] + + @Option(name: .long, help: ArgumentHelp("Add a host-to-IP mapping resolvable during the build", valueName: "host=ip")) + var addHost: [String] = [] + + @Option(name: .long, help: ArgumentHelp("Hostname the build environment reports", valueName: "name")) + var hostname: String? + + @Option(name: .long, help: ArgumentHelp("Size of /dev/shm during the build", valueName: "bytes")) + var shmSize: String? + + @Option(name: .long, help: ArgumentHelp("Set a resource limit for the build", valueName: "type=soft:hard")) + var ulimit: [String] = [] + + @Option(name: .long, help: ArgumentHelp("Parent cgroup for the build environment", valueName: "cgroup")) + var cgroupParent: String? + + @Option(name: .long, help: ArgumentHelp("Network mode for the build (none, host or sandbox)", valueName: "mode")) + var network: String? + @Option(name: .long, help: ArgumentHelp("Cache imports for the build", valueName: "value", visibility: .hidden)) var cacheIn: [String] = { [] @@ -81,6 +103,10 @@ extension Application { var dockerfile: String = "-" + /// `--shm-size` parsed to bytes. The frontend takes a plain byte count, + /// so the suffixed forms users expect from docker are resolved here. + var shmSizeBytes: UInt64? + @Option(name: .shortAndLong, help: ArgumentHelp("Set a label", valueName: "key=val")) var label: [String] = [] @@ -93,9 +119,9 @@ extension Application { @Flag(name: .long, help: "Do not use cache") var noCache: Bool = false - @Option(name: .shortAndLong, help: ArgumentHelp("Output configuration for the build (format: type=[,dest=])", valueName: "value")) + @Option(name: .shortAndLong, help: ArgumentHelp("Output configuration for the build (format: type=[,dest=])", valueName: "value")) var output: [String] = { - ["type=oci"] + ["type=store"] }() @Option( @@ -156,6 +182,24 @@ extension Application { public func run() async throws { let containerSystemConfig: ContainerSystemConfig = try await Application.loadContainerSystemConfig() + // The blobs an image export writes gather in an ingest session + // rather than landing in the store as they arrive. The store takes + // the whole set at the moment it records the images naming them, + // so it never holds a blob that no image claims and a sweep + // running alongside can collect. + // + // Only an export into the store gathers, and that export is what + // hands the gathering over, so a build that reaches its end has + // nothing left to give back and a build that does not is what the + // catch returns it for. + // + // Whether any export lands in the store is asked of the exports + // themselves. An output the exports cannot be read from is + // gathered for, and read again below where a build reports what is + // wrong with what it was given. + let contentStore = RemoteContentStoreClient() + let exportsToStore = (try? output.contains { try Builder.BuildExport(from: $0).type == "store" }) ?? true + let ingest = exportsToStore ? try await contentStore.newIngestSession() : nil do { let timeout: Duration = .seconds(300) let progressConfig = try ProgressConfig( @@ -370,13 +414,22 @@ extension Application { }() group.addTask { [ - terminal, buildArg, secretsData, ssh, contextDir, ignoreFileData, label, noCache, target, quiet, cacheIn, cacheOut, pull, exports, imageNames, tempURL, - log + terminal, buildArg, buildContext, addHost, hostname, shmSizeBytes, ulimit, cgroupParent, network, + secretsData, ssh, contextDir, ignoreFileData, label, noCache, target, quiet, cacheIn, cacheOut, pull, exports, + imageNames, tempURL, log, contentStore, ingest, ] in let config = Builder.BuildConfig( buildID: buildID, - contentStore: RemoteContentStoreClient(), + contentStore: contentStore, + ingestDir: ingest?.ingestDir, buildArgs: buildArg, + buildContexts: buildContext, + addHosts: addHost, + ulimits: ulimit, + hostname: hostname, + shmSize: shmSizeBytes, + cgroupParent: cgroupParent, + network: network, secrets: secretsData, ssh: ssh, contextDir: contextDir, @@ -418,6 +471,33 @@ extension Application { unpackProgress.add(tasks: 1) let unpackTask = await taskManager.startTask() switch exp.type { + case "store": + // The build gathered its blobs rather than + // handing them over as they arrived; what + // remains is the root digest the builder left + // on the export path, and the store takes the + // gathering and records every tag naming it + // together, so the content is claimed from the + // moment the store holds it. + try Task.checkCancellation() + let digestURL = tempURL.appendingPathComponent("digest") + let digest = try String(contentsOf: digestURL, encoding: .utf8) + .trimmingCharacters(in: .whitespacesAndNewlines) + guard let ingest else { + throw ContainerizationError( + .internalError, + message: "export into the store gathered nowhere" + ) + } + let images = try await ClientImage.createFromIngest( + ingestSession: ingest.id, + references: imageNames, + rootDigest: digest + ) + for image in images { + try Task.checkCancellation() + try await image.unpack(platform: nil, progressUpdate: ProgressTaskCoordinator.handler(for: unpackTask, from: unpackProgress.handler)) + } case "oci": try Task.checkCancellation() guard let dest = exp.destination else { @@ -468,6 +548,9 @@ extension Application { try await group.next() } } catch { + if let ingest { + try? await contentStore.cancelIngestSession(ingest.id) + } throw NSError(domain: "Build", code: 1, userInfo: [NSLocalizedDescriptionKey: "\(error)"]) } } @@ -483,6 +566,88 @@ extension Application { } } + // Each --build-context is name=value, where the value is either a + // non-local reference (image, git, URL, oci-layout) passed through + // to the builder, or a local directory. A local directory must + // exist, and is resolved to an absolute path here so that neither + // the API server's nor the builder's working directory can change + // its meaning. + let passthroughPrefixes = [ + "docker-image://", "oci-layout:", "http://", "https://", + "git://", "git@", "ssh://", "local:", "input:", + ] + buildContext = try buildContext.map { entry in + let parts = entry.split(separator: "=", maxSplits: 1) + guard parts.count == 2, !parts[0].isEmpty, !parts[1].isEmpty else { + throw ValidationError("build context must be name=value: \(entry)") + } + let value = String(parts[1]) + if passthroughPrefixes.contains(where: { value.hasPrefix($0) }) { + // An oci-layout value names a layout directory on this + // machine, so a missing directory can be refused here with + // its real cause; anything that slips past resolves as an + // image reference and fails with a misleading + // invalid-domain error. + if value.hasPrefix("oci-layout://") { + var path = String(value.dropFirst("oci-layout://".count)) + if let at = path.firstIndex(of: "@") { + path = String(path[path.startIndex.. ClientImage { + let client = newXPCClient() + let request = newRequest(.imageCreate) + let description = ImageDescription(reference: reference, descriptor: descriptor) + request.set(key: .imageDescription, value: try JSONEncoder().encode(description)) + let reply = try await client.send(request) + + guard let data = reply.dataNoCopy(key: .imageDescription) else { + throw ContainerizationError(.internalError, message: "no image description returned for \(reference)") + } + let created = try JSONDecoder().decode(ImageDescription.self, from: data) + return ClientImage(description: created) + } + + /// Land the content an ingest session holds and register the images that + /// name it, in one operation the store performs under a single hold of its + /// lock, so no sweep runs between the content arriving and the records + /// claiming it. The root descriptor is read on the far side, where the + /// content becomes readable. + public static func createFromIngest(ingestSession: String, references: [String], rootDigest: String) async throws -> [ClientImage] { + let client = newXPCClient() + let request = newRequest(.imageCreateFromIngest) + request.set(key: .ingestSessionId, value: ingestSession) + request.set(key: .digest, value: rootDigest) + request.set(key: .imageReferences, value: try JSONEncoder().encode(references)) + let reply = try await client.send(request) + + guard let data = reply.dataNoCopy(key: .imageDescriptions) else { + throw ContainerizationError(.internalError, message: "no image descriptions returned for \(references.joined(separator: ", "))") + } + let created = try JSONDecoder().decode([ImageDescription].self, from: data) + return created.map { ClientImage(description: $0) } + } + public static func load(from tarFile: String, force: Bool = false) async throws -> ImageLoadResult { let client = newXPCClient() let request = newRequest(.imageLoad) diff --git a/Sources/Services/ContainerImagesService/Client/ImageServiceXPCKeys.swift b/Sources/Services/ContainerImagesService/Client/ImageServiceXPCKeys.swift index c087f99f1..383e8bbc5 100644 --- a/Sources/Services/ContainerImagesService/Client/ImageServiceXPCKeys.swift +++ b/Sources/Services/ContainerImagesService/Client/ImageServiceXPCKeys.swift @@ -28,6 +28,7 @@ public enum ImagesServiceXPCKeys: String { /// Images case imageReference + case imageReferences case imageNewReference case imageDescription case imageDescriptions diff --git a/Sources/Services/ContainerImagesService/Client/ImageServiceXPCRoutes.swift b/Sources/Services/ContainerImagesService/Client/ImageServiceXPCRoutes.swift index a381f8e74..872b55028 100644 --- a/Sources/Services/ContainerImagesService/Client/ImageServiceXPCRoutes.swift +++ b/Sources/Services/ContainerImagesService/Client/ImageServiceXPCRoutes.swift @@ -24,6 +24,8 @@ public enum ImagesServiceXPCRoute: String { case imagePush case imageTag case imageBuild + case imageCreate + case imageCreateFromIngest case imageDelete case imageSave case imageLoad diff --git a/Sources/Services/ContainerImagesService/Server/ImagesService.swift b/Sources/Services/ContainerImagesService/Server/ImagesService.swift index 21a3e5146..69deac39e 100644 --- a/Sources/Services/ContainerImagesService/Server/ImagesService.swift +++ b/Sources/Services/ContainerImagesService/Server/ImagesService.swift @@ -32,6 +32,17 @@ public actor ImagesService { private let imageStore: ImageStore private let snapshotStore: SnapshotStore + /// One landing, removal, or sweep at a time. The store operations + /// suspend mid-flight, and the actor admits other calls at every + /// suspension, so a garbage collection overlapping a landing computes + /// its keep set without the arriving image and prunes blobs and + /// snapshots the landing is about to reference. The lock is held + /// across each whole mutating operation, the way PodsService holds + /// its lock across lifecycle operations; containerd guards the same + /// window with leases. + /// https://github.com/containerd/containerd/blob/main/docs/garbage-collection.md + private let lock = AsyncLock() + public init( contentStore: ContentStore, imageStore: ImageStore, @@ -214,6 +225,109 @@ public actor ImagesService { try writer.finishEncoding() } + /// Register an image whose content is already in the store, under the + /// reference the description names. This is how a built image arrives + /// when its blobs were exported straight into the content store: the + /// root descriptor is all that remains to record. + public func create(description: ImageDescription) async throws -> ImageDescription { + try await lock.withLock { _ in + try await self._create(description: description) + } + } + + /// Land the content an ingest session holds and register the images that + /// name it, under one hold of the lock. + /// + /// A blob is claimed by nobody between arriving in the store and a record + /// naming it, and a sweep that runs in that window takes it. Content + /// written into an ingest session is not in the store yet and no sweep + /// lists it, so completing the session and recording the images together + /// leaves no such window: the blobs appear at a moment when no sweep can + /// run, and the records claiming them are there when the lock is released. + /// Pulling an image and loading an OCI layout both land their content this + /// way. + /// + /// The root descriptor is read here rather than taken from the caller, + /// because the content it describes is unreadable until the session + /// completes, which happens inside this lock. + public func createFromIngest(ingestSession: String, references: [String], rootDigest: String) async throws -> [ImageDescription] { + try await lock.withLock { _ in + self.log.debug( + "ImagesService: enter", + metadata: [ + "func": "\(#function)", + "session": "\(ingestSession)", + "digest": "\(rootDigest)", + "references": "\(references.joined(separator: ", "))", + ] + ) + defer { + self.log.debug( + "ImagesService: exit", + metadata: ["func": "\(#function)", "session": "\(ingestSession)"] + ) + } + + _ = try await self.contentStore.completeIngestSession(ingestSession) + guard let content: Content = try await self.contentStore.get(digest: rootDigest) else { + throw ContainerizationError(.notFound, message: "built image root \(rootDigest) not in the content store") + } + let index = try content.decode() as ContainerizationOCI.Index + let descriptor = Descriptor( + mediaType: index.mediaType, + digest: rootDigest, + size: Int64(try content.size()) + ) + var created: [ImageDescription] = [] + for reference in references { + created.append( + try await self._create(description: ImageDescription(reference: reference, descriptor: descriptor)) + ) + } + return created + } + } + + private func _create(description: ImageDescription) async throws -> ImageDescription { + self.log.debug( + "ImagesService: enter", + metadata: [ + "func": "\(#function)", + "reference": "\(description.reference)", + "digest": "\(description.descriptor.digest)", + ] + ) + defer { + self.log.debug( + "ImagesService: exit", + metadata: [ + "func": "\(#function)", + "reference": "\(description.reference)", + ] + ) + } + + let image = try await self.imageStore.create(description: description.toCZ) + // Creating over an existing reference replaces its record and leaves + // the replaced generation's unpacked snapshot owned by no image, so + // the create sweeps the snapshot directories no current image claims. + // The sweep is housekeeping: its failure warns in the log and never + // fails the create that carried it. + do { + let kept = try await self._list() + _ = try await self.snapshotStore.clean(keepingSnapshotsFor: kept) + } catch { + self.log.warning( + "ImagesService: snapshot sweep failed", + metadata: [ + "func": "\(#function)", + "error": "\(error)", + ] + ) + } + return image.description.fromCZ + } + public func load(from tarFile: URL, force: Bool) async throws -> ([ImageDescription], [String]) { let archivePathname = tarFile.absolutePath() self.log.debug( diff --git a/Sources/Services/ContainerImagesService/Server/ImagesServiceHarness.swift b/Sources/Services/ContainerImagesService/Server/ImagesServiceHarness.swift index e88b2368f..00942bdc5 100644 --- a/Sources/Services/ContainerImagesService/Server/ImagesServiceHarness.swift +++ b/Sources/Services/ContainerImagesService/Server/ImagesServiceHarness.swift @@ -159,6 +159,40 @@ public struct ImagesServiceHarness: Sendable { return reply } + @Sendable + public func create(_ message: XPCMessage) async throws -> XPCMessage { + guard let data = message.dataNoCopy(key: .imageDescription) else { + throw ContainerizationError( + .invalidArgument, + message: "missing image description" + ) + } + let description = try JSONDecoder().decode(ImageDescription.self, from: data) + let created = try await service.create(description: description) + let reply = message.reply() + reply.set(key: .imageDescription, value: try JSONEncoder().encode(created)) + return reply + } + + @Sendable + public func createFromIngest(_ message: XPCMessage) async throws -> XPCMessage { + guard let session = message.string(key: .ingestSessionId) else { + throw ContainerizationError(.invalidArgument, message: "missing ingest session id") + } + guard let digest = message.string(key: .digest) else { + throw ContainerizationError(.invalidArgument, message: "missing root digest") + } + guard let data = message.dataNoCopy(key: .imageReferences) else { + throw ContainerizationError(.invalidArgument, message: "missing image references") + } + let references = try JSONDecoder().decode([String].self, from: data) + let created = try await service.createFromIngest( + ingestSession: session, references: references, rootDigest: digest) + let reply = message.reply() + reply.set(key: .imageDescriptions, value: try JSONEncoder().encode(created)) + return reply + } + @Sendable public func load(_ message: XPCMessage) async throws -> XPCMessage { let input = message.string(key: .filePath) diff --git a/Tests/ContainerBuildTests/BuildFSSyncTests.swift b/Tests/ContainerBuildTests/BuildFSSyncTests.swift index 5d6d607d2..55012ea66 100644 --- a/Tests/ContainerBuildTests/BuildFSSyncTests.swift +++ b/Tests/ContainerBuildTests/BuildFSSyncTests.swift @@ -394,4 +394,105 @@ import Testing let secretLeak = infos.first { $0.name.hasSuffix("secret.txt") } #expect(secretLeak == nil, "no entry for the external file should appear in walk() results: \(infos.map { $0.name })") } + + // MARK: - named contexts: root selection and containment + // + // Every serving path resolves its root through contextRoot(): the dir-name + // the shim relays either names a declared named context, is one of the two + // primary names ("context", "dockerfile"), or is refused. These tests pin + // that mapping and show the boundary enforcement applies under a named + // root exactly as it does under the primary one. + + /// Returns a read packet whose dir-name metadata selects a context root. + private func readPacket(source: String, dirName: String) -> BuildTransfer { + var p = readPacket(source: source) + p.metadata["dir-name"] = dirName + return p + } + + @Test func testInfoServesFromNamedContextRoot() async throws { + let namedDir = base.appendingPathComponent("dep") + try fm.createDirectory(at: namedDir, withIntermediateDirectories: true) + try write("hello", to: namedDir.appendingPathComponent("hello.txt")) + + let fssync = try BuildFSSync(contextDir, namedContexts: ["dep": namedDir]) + var continuation: AsyncStream.Continuation! + _ = AsyncStream { continuation = $0 } + defer { continuation.finish() } + + // hello.txt exists only under the named root; resolving it proves the + // dir-name selected that root rather than the primary context. + try await fssync.info(continuation, readPacket(source: "hello.txt", dirName: "dep"), "build-0") + } + + @Test func testInfoPrimaryNamesResolveToContextDir() async throws { + let namedDir = base.appendingPathComponent("dep") + try fm.createDirectory(at: namedDir, withIntermediateDirectories: true) + try write("primary", to: contextDir.appendingPathComponent("main.txt")) + + let fssync = try BuildFSSync(contextDir, namedContexts: ["dep": namedDir]) + var continuation: AsyncStream.Continuation! + _ = AsyncStream { continuation = $0 } + defer { continuation.finish() } + + // main.txt exists only in the primary context; both reserved names and + // the no-metadata packet must resolve there even with named contexts + // declared. + try await fssync.info(continuation, readPacket(source: "main.txt", dirName: "context"), "build-0") + try await fssync.info(continuation, readPacket(source: "main.txt", dirName: "dockerfile"), "build-0") + try await fssync.info(continuation, readPacket(source: "main.txt"), "build-0") + } + + @Test func testReadRejectsUndeclaredNamedContext() async throws { + try write("primary", to: contextDir.appendingPathComponent("main.txt")) + + let fssync = try BuildFSSync(contextDir) + var continuation: AsyncStream.Continuation! + _ = AsyncStream { continuation = $0 } + defer { continuation.finish() } + + do { + try await fssync.read(continuation, readPacket(source: "main.txt", dirName: "undeclared"), "build-0") + Issue.record("read() should refuse a dir-name that was never declared on the command line") + } catch let error as BuildFSSync.Error { + #expect(error == .unknownNamedContext("undeclared")) + } + } + + @Test func testReadRejectsSymlinkEscapeFromNamedContextRoot() async throws { + let namedDir = base.appendingPathComponent("dep") + try fm.createDirectory(at: namedDir, withIntermediateDirectories: true) + let secretFile = outsideDir.appendingPathComponent("secret.txt") + try write("supersecret", to: secretFile) + + // Symlink inside the named root → absolute path outside it. The + // primary-context tests above prove this is refused under contextDir; + // this proves the same enforcement holds under a named root. + try fm.createSymbolicLink( + atPath: namedDir.appendingPathComponent("leak").path(percentEncoded: false), + withDestinationPath: secretFile.path(percentEncoded: false) + ) + + let fssync = try BuildFSSync(contextDir, namedContexts: ["dep": namedDir]) + var continuation: AsyncStream.Continuation! + _ = AsyncStream { continuation = $0 } + defer { continuation.finish() } + + do { + try await fssync.read(continuation, readPacket(source: "leak", dirName: "dep"), "build-0") + Issue.record("read() should throw BuildFSSync.Error for a symlink that resolves outside its named context root") + } catch is BuildFSSync.Error { + // expected + } + } + + @Test func testInitRejectsMissingNamedContextDirectory() async throws { + let missing = base.appendingPathComponent("never-created") + do { + _ = try BuildFSSync(contextDir, namedContexts: ["dep": missing]) + Issue.record("init should refuse a named context directory that does not exist") + } catch is BuildFSSync.Error { + // expected + } + } } diff --git a/Tests/ContainerBuildTests/BuildRemoteContentWriteTests.swift b/Tests/ContainerBuildTests/BuildRemoteContentWriteTests.swift new file mode 100644 index 000000000..eef9b6df4 --- /dev/null +++ b/Tests/ContainerBuildTests/BuildRemoteContentWriteTests.swift @@ -0,0 +1,96 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationError +import CryptoKit +import Foundation +import Testing + +@testable import ContainerBuild + +// Tests for the write sessions the content-store stage keeps, one per blob the +// builder streams in, each holding the bytes and the running digest until the +// commit seals them. +@Suite class BuildRemoteContentWriteTests { + private static func digest(of data: Data) -> String { + "sha256:" + SHA256.hash(data: data).map { String(format: "%02x", $0) }.joined() + } + + @Test func testWriteAtZeroOffsetRestartsTheBlob() async throws { + let sessions = BuildRemoteContentProxy.WriteSessions() + let abandoned = Data(repeating: 0xaa, count: 32) + let kept = Data("the bytes the commit seals".utf8) + + let held = try await sessions.append(ref: "layer", offset: 0, data: abandoned) + #expect(held == UInt64(abandoned.count)) + + let restarted = try await sessions.append(ref: "layer", offset: 0, data: kept) + #expect(restarted == UInt64(kept.count)) + + let result = try await sessions.close(ref: "layer") + let closed = try #require(result) + #expect(closed.written == UInt64(kept.count)) + #expect(closed.digest == Self.digest(of: kept)) + + let onDisk = try Data(contentsOf: closed.url) + #expect(onDisk == kept) + + try? FileManager.default.removeItem(at: closed.url) + } + + @Test func testWriteAtAnOffsetTheSessionIsNotAtFails() async throws { + let sessions = BuildRemoteContentProxy.WriteSessions() + let held = try await sessions.append(ref: "layer", offset: 0, data: Data(repeating: 0xaa, count: 32)) + #expect(held == 32) + + await #expect(throws: ContainerizationError.self) { + _ = try await sessions.append(ref: "layer", offset: 8, data: Data([0x01])) + } + + let result = try await sessions.close(ref: "layer") + let closed = try #require(result) + try? FileManager.default.removeItem(at: closed.url) + } + + @Test func testDiscardingLetsGoOfWhatWasNeverCommitted() async throws { + let sessions = BuildRemoteContentProxy.WriteSessions() + _ = try await sessions.append(ref: "layer", offset: 0, data: Data(repeating: 0xaa, count: 32)) + let dir = await sessions.dir + #expect(FileManager.default.fileExists(atPath: dir.path)) + + await sessions.discardAll() + + #expect(!FileManager.default.fileExists(atPath: dir.path)) + let afterDiscard = try await sessions.close(ref: "layer") + #expect(afterDiscard == nil) + } + + @Test func testABlobWrittenAfterDiscardingStartsAgain() async throws { + let sessions = BuildRemoteContentProxy.WriteSessions() + _ = try await sessions.append(ref: "layer", offset: 0, data: Data(repeating: 0xaa, count: 32)) + await sessions.discardAll() + + let kept = Data("written after the directory went away".utf8) + let written = try await sessions.append(ref: "layer", offset: 0, data: kept) + #expect(written == UInt64(kept.count)) + + let result = try await sessions.close(ref: "layer") + let closed = try #require(result) + #expect(closed.digest == Self.digest(of: kept)) + + await sessions.discardAll() + } +}