From 7a55373e1a8d737be55dc966602edf486c1b2a9b Mon Sep 17 00:00:00 2001 From: JP Simard Date: Wed, 2 Mar 2022 21:24:43 -0500 Subject: [PATCH 1/5] Add Unxip tool from saagarjha Source: https://github.com/saagarjha/unxip This mirrors this change in Xcodes.app: https://github.com/RobotsAndPencils/XcodesApp/pull/179 In my tests (M1 Max, 64GB RAM), unxipping is 3x faster than `/usr/bin/xip`. --- .../xcschemes/xcodes-Package.xcscheme | 28 + Package.swift | 1 + Sources/Unxip/Unxip.swift | 565 ++++++++++++++++++ 3 files changed, 594 insertions(+) create mode 100644 Sources/Unxip/Unxip.swift diff --git a/.swiftpm/xcode/xcshareddata/xcschemes/xcodes-Package.xcscheme b/.swiftpm/xcode/xcshareddata/xcschemes/xcodes-Package.xcscheme index f2745c4..165d010 100644 --- a/.swiftpm/xcode/xcshareddata/xcschemes/xcodes-Package.xcscheme +++ b/.swiftpm/xcode/xcshareddata/xcschemes/xcodes-Package.xcscheme @@ -90,6 +90,34 @@ ReferencedContainer = "container:"> + + + + + + + + SubSequence { + let toOffset = toOffset ?? count + return self[index(startIndex, offsetBy: fromOffset).. SubSequence { + let base = index(startIndex, offsetBy: fromOffset) + return self[base.. { + let batchSize: Int + var operations = [@Sendable () async throws -> TaskResult]() + + var results: AsyncStream { + AsyncStream(bufferingPolicy: .bufferingOldest(batchSize)) { continuation in + Task { + try await withThrowingTaskGroup(of: (Int, TaskResult).self) { group in + var queueIndex = 0 + var dequeIndex = 0 + var pending = [Int: TaskResult]() + while dequeIndex < operations.count { + if queueIndex - dequeIndex < batchSize, + queueIndex < operations.count + { + let _queueIndex = queueIndex + group.addTask { + let queueIndex = _queueIndex + return await (queueIndex, try operations[queueIndex]()) + } + queueIndex += 1 + } else { + let (index, result) = try await group.next()! + pending[index] = result + if index == dequeIndex { + while let result = pending[dequeIndex] { + await continuation.yieldWithBackoff(result) + pending.removeValue(forKey: dequeIndex) + dequeIndex += 1 + } + } + } + } + continuation.finish() + } + } + } + } + + init(batchSize: Int = ProcessInfo.processInfo.activeProcessorCount) { + self.batchSize = batchSize + } + + mutating func addTask(operation: @escaping @Sendable () async throws -> TaskResult) { + operations.append(operation) + } + + mutating func addRunningTask(operation: @escaping @Sendable () async -> TaskResult) -> Task { + let task = Task { + await operation() + } + operations.append({ + await task.value + }) + return task + } +} + +final class Chunk: Sendable { + let buffer: UnsafeBufferPointer + let owned: Bool + + init(buffer: UnsafeBufferPointer, owned: Bool) { + self.buffer = buffer + self.owned = owned + } + + deinit { + if owned { + buffer.deallocate() + } + } +} + +struct File { + let dev: Int + let ino: Int + let mode: Int + let name: String + var data = [UnsafeBufferPointer]() + // For keeping the data alive + var chunks = [Chunk]() + + struct Identifier: Hashable { + let dev: Int + let ino: Int + } + + var identifier: Identifier { + Identifier(dev: dev, ino: ino) + } + + func writeCompressedIfPossible(usingDescriptor descriptor: CInt) async -> Bool { + let blockSize = 64 << 10 // LZFSE with 64K block size + var _data = [UInt8]() + _data.reserveCapacity(self.data.map(\.count).reduce(0, +)) + let data = self.data.reduce(into: _data, +=) + var compressionStream = ConcurrentStream<[UInt8]?>() + var position = data.startIndex + + while position < data.endIndex { + let _position = position + compressionStream.addTask { + try Task.checkCancellation() + let position = _position + let data = [UInt8](unsafeUninitializedCapacity: blockSize + blockSize / 16) { buffer, count in + data[position...size + let size = tableSize + chunks.map(\.count).reduce(0, +) + guard size < data.count else { + return false + } + + let buffer = [UInt8](unsafeUninitializedCapacity: size) { buffer, count in + var position = tableSize + + func writePosition(toTableIndex index: Int) { + precondition(position < UInt32.max) + for i in 0...size { + buffer[index * MemoryLayout.size + i] = UInt8(position >> (i * 8) & 0xff) + } + } + + writePosition(toTableIndex: 0) + for (index, chunk) in zip(1..., chunks) { + _ = UnsafeMutableBufferPointer(rebasing: buffer.suffix(from: position)).initialize(from: chunk) + position += chunk.count + writePosition(toTableIndex: index) + } + count = size + } + + let attribute = + "cmpf".utf8.reversed() // magic + + [0x0c, 0x00, 0x00, 0x00] // LZFSE, 64K chunks + + ([ + (data.count >> 0) & 0xff, + (data.count >> 8) & 0xff, + (data.count >> 16) & 0xff, + (data.count >> 24) & 0xff, + (data.count >> 32) & 0xff, + (data.count >> 40) & 0xff, + (data.count >> 48) & 0xff, + (data.count >> 56) & 0xff, + ].map(UInt8.init) as [UInt8]) + + guard fsetxattr(descriptor, "com.apple.decmpfs", attribute, attribute.count, 0, XATTR_SHOWCOMPRESSION) == 0 else { + return false + } + + let resourceForkDescriptor = open(name + _PATH_RSRCFORKSPEC, O_WRONLY | O_CREAT, 0o666) + guard resourceForkDescriptor >= 0 else { + return false + } + defer { + close(resourceForkDescriptor) + } + + var written: Int + repeat { + // TODO: handle partial writes smarter + written = pwrite(resourceForkDescriptor, buffer, buffer.count, 0) + guard written >= 0 else { + return false + } + } while written != buffer.count + + guard fchflags(descriptor, UInt32(UF_COMPRESSED)) == 0 else { + return false + } + + return true + } +} + +extension option { + init(name: StaticString, has_arg: CInt, flag: UnsafeMutablePointer?, val: StringLiteralType) { + let _option = name.withUTF8Buffer { + $0.withMemoryRebound(to: CChar.self) { + option(name: $0.baseAddress, has_arg: has_arg, flag: flag, val: CInt(UnicodeScalar(val)!.value)) + } + } + self = _option + } +} + +struct Options { + var input: URL + var output: URL? + var compress: Bool = true + + init() { + let options = [ + option(name: "compression-disable", has_arg: no_argument, flag: nil, val: "c"), + option(name: "help", has_arg: no_argument, flag: nil, val: "h"), + option(name: nil, has_arg: 0, flag: nil, val: 0), + ] + var result: CInt + repeat { + result = getopt_long(CommandLine.argc, CommandLine.unsafeArgv, "ch", options, nil) + if result < 0 { + break + } + switch UnicodeScalar(UInt32(result)) { + case "c": + compress = false + case "h": + Self.printUsage(nominally: true) + default: + Self.printUsage(nominally: false) + } + } while true + + let arguments = UnsafeBufferPointer(start: CommandLine.unsafeArgv + Int(optind), count: Int(CommandLine.argc - optind)).map { + String(cString: $0!) + } + + guard let input = arguments.first else { + Self.printUsage(nominally: false) + } + + self.input = URL(fileURLWithPath: input) + + guard let output = arguments.dropFirst().first else { + return + } + + self.output = URL(fileURLWithPath: output) + } + + static func printUsage(nominally: Bool) -> Never { + fputs( + """ + A fast Xcode unarchiver + + USAGE: unxip [options] [output] + + OPTIONS: + -c, --compression-disable Disable APFS compression of result. + -h, --help Print this help message. + """, nominally ? stdout : stderr) + exit(nominally ? EXIT_SUCCESS : EXIT_FAILURE) + } +} + +@main +struct Main { + static let options = Options() + + static func read(_ type: Integer.Type, from buffer: inout Buffer) -> Integer where Buffer.Element == UInt8, Buffer.SubSequence == Buffer { + defer { + buffer = buffer[fromOffset: MemoryLayout.size] + } + var result: Integer = 0 + var iterator = buffer.makeIterator() + for _ in 0...size { + result <<= 8 + result |= Integer(iterator.next()!) + } + return result + } + + static func chunks(from content: UnsafeBufferPointer) -> ConcurrentStream { + var remaining = content[fromOffset: 4] + let chunkSize = read(UInt64.self, from: &remaining) + var decompressedSize: UInt64 = 0 + + var chunkStream = ConcurrentStream() + + repeat { + decompressedSize = read(UInt64.self, from: &remaining) + let compressedSize = read(UInt64.self, from: &remaining) + let _remaining = remaining + let _decompressedSize = decompressedSize + + chunkStream.addTask { + let remaining = _remaining + let decompressedSize = _decompressedSize + + if compressedSize == chunkSize { + return Chunk(buffer: UnsafeBufferPointer(rebasing: remaining[fromOffset: 0, size: Int(compressedSize)]), owned: false) + } else { + let magic = [0xfd] + "7zX".utf8 + precondition(remaining.prefix(magic.count).elementsEqual(magic)) + let buffer = UnsafeMutableBufferPointer.allocate(capacity: Int(decompressedSize)) + precondition(compression_decode_buffer(buffer.baseAddress!, buffer.count, UnsafeBufferPointer(rebasing: remaining).baseAddress!, Int(compressedSize), nil, COMPRESSION_LZMA) == decompressedSize) + return Chunk(buffer: UnsafeBufferPointer(buffer), owned: true) + } + } + remaining = remaining[fromOffset: Int(compressedSize)] + } while decompressedSize == chunkSize + + return chunkStream + } + + static func files(in chunkStream: ChunkStream) -> AsyncStream where ChunkStream.Element == Chunk { + AsyncStream(bufferingPolicy: .bufferingOldest(ProcessInfo.processInfo.activeProcessorCount)) { continuation in + Task { + var iterator = chunkStream.makeAsyncIterator() + var chunk = try! await iterator.next()! + var position = 0 + + func read(size: Int) async -> [UInt8] { + var result = [UInt8]() + while result.count < size { + if position >= chunk.buffer.endIndex { + chunk = try! await iterator.next()! + position = 0 + } + result.append(chunk.buffer[chunk.buffer.startIndex + position]) + position += 1 + } + return result + } + + func readOctal(from bytes: [UInt8]) -> Int { + Int(String(data: Data(bytes), encoding: .utf8)!, radix: 8)! + } + + while true { + let magic = await read(size: 6) + // Yes, cpio.h really defines this global macro + precondition(magic.elementsEqual(MAGIC.utf8)) + let dev = readOctal(from: await read(size: 6)) + let ino = readOctal(from: await read(size: 6)) + let mode = readOctal(from: await read(size: 6)) + let _ = await read(size: 6) // uid + let _ = await read(size: 6) // gid + let _ = await read(size: 6) // nlink + let _ = await read(size: 6) // rdev + let _ = await read(size: 11) // mtime + let namesize = readOctal(from: await read(size: 6)) + var filesize = readOctal(from: await read(size: 11)) + let name = String(cString: await read(size: namesize)) + var file = File(dev: dev, ino: ino, mode: mode, name: name) + + while filesize > 0 { + if position >= chunk.buffer.endIndex { + chunk = try! await iterator.next()! + position = 0 + } + let size = min(filesize, chunk.buffer.endIndex - position) + file.chunks.append(chunk) + file.data.append(UnsafeBufferPointer(rebasing: chunk.buffer[fromOffset: position, size: size])) + filesize -= size + position += size + } + + guard file.name != "TRAILER!!!" else { + continuation.finish() + return + } + + await continuation.yieldWithBackoff(file) + } + } + } + } + + static func parseContent(_ content: UnsafeBufferPointer) async { + var taskStream = ConcurrentStream(batchSize: 64) // Worst case, should allow for files up to 64 * 16MB = 1GB + var hardlinks = [File.Identifier: (String, Task)]() + var directories = [Substring: Task]() + for await file in files(in: chunks(from: content).results) { + @Sendable + func warn(_ result: CInt, _ operation: String) { + if result != 0 { + perror("\(operation) \(file.name) failed") + } + } + + // The assumption is that all directories are provided without trailing slashes + func parentDirectory(of path: S) -> S.SubSequence { + return path[.. Task? { + directories[parentDirectory(of: file.name)] ?? directories[String(parentDirectory(of: file.name))[...]] + } + + @Sendable + func setStickyBit(on file: File) { + if file.mode & Int(C_ISVTX) != 0 { + warn(chmod(file.name, mode_t(file.mode)), "Setting sticky bit on") + } + } + + if file.name == "." { + continue + } + + if let (original, task) = hardlinks[file.identifier] { + _ = taskStream.addRunningTask { + await task.value + warn(link(original, file.name), "linking") + } + continue + } + + // The types we care about, anyways + let typeMask = Int(C_ISLNK | C_ISDIR | C_ISREG) + switch CInt(file.mode & typeMask) { + case C_ISLNK: + let task = parentDirectoryTask(for: file) + assert(task != nil, file.name) + _ = taskStream.addRunningTask { + warn(symlink(String(data: Data(file.data.map(Array.init).reduce([], +)), encoding: .utf8), file.name), "symlinking") + setStickyBit(on: file) + } + case C_ISDIR: + let task = parentDirectoryTask(for: file) + assert(task != nil || parentDirectory(of: file.name) == ".", file.name) + directories[file.name[...]] = taskStream.addRunningTask { + await task?.value + warn(mkdir(file.name, mode_t(file.mode & 0o777)), "creating directory at") + setStickyBit(on: file) + } + case C_ISREG: + let task = parentDirectoryTask(for: file) + assert(task != nil, file.name) + hardlinks[file.identifier] = ( + file.name, + taskStream.addRunningTask { + await task?.value + + let fd = open(file.name, O_CREAT | O_WRONLY, mode_t(file.mode & 0o777)) + if fd < 0 { + warn(fd, "creating file at") + return + } + defer { + warn(close(fd), "closing") + setStickyBit(on: file) + } + + if options.compress, + await file.writeCompressedIfPossible(usingDescriptor: fd) + { + return + } + + // pwritev requires the vector count to be positive + if file.data.count == 0 { + return + } + + var vector = file.data.map { + iovec(iov_base: UnsafeMutableRawPointer(mutating: $0.baseAddress), iov_len: $0.count) + } + let total = file.data.map(\.count).reduce(0, +) + var written = 0 + + repeat { + // TODO: handle partial writes smarter + written = pwritev(fd, &vector, CInt(vector.count), 0) + if written < 0 { + warn(-1, "writing chunk to") + break + } + } while written != total + } + ) + default: + fatalError("\(file.name) with \(file.mode) is a type that is unhandled") + } + } + + // Run through any stragglers + for await _ in taskStream.results { + } + } + + static func locateContent(in file: UnsafeBufferPointer) -> UnsafeBufferPointer { + precondition(file.starts(with: "xar!".utf8)) // magic + var header = file[4...] + let headerSize = read(UInt16.self, from: &header) + precondition(read(UInt16.self, from: &header) == 1) // version + let tocCompressedSize = read(UInt64.self, from: &header) + let tocDecompressedSize = read(UInt64.self, from: &header) + _ = read(UInt32.self, from: &header) // checksum + + let toc = [UInt8](unsafeUninitializedCapacity: Int(tocDecompressedSize)) { buffer, count in + let zlibSkip = 2 // Apple's decoder doesn't want to see CMF/FLG (see RFC 1950) + count = compression_decode_buffer(buffer.baseAddress!, Int(tocDecompressedSize), file.baseAddress! + Int(headerSize) + zlibSkip, Int(tocCompressedSize) - zlibSkip, nil, COMPRESSION_ZLIB) + precondition(count == Int(tocDecompressedSize)) + } + + let document = try! XMLDocument(data: Data(toc)) + let content = try! document.nodes(forXPath: "xar/toc/file").first { + try! $0.nodes(forXPath: "name").first!.stringValue! == "Content" + }! + let contentOffset = Int(try! content.nodes(forXPath: "data/offset").first!.stringValue!)! + let contentSize = Int(try! content.nodes(forXPath: "data/length").first!.stringValue!)! + let contentBase = Int(headerSize) + Int(tocCompressedSize) + contentOffset + + let slice = file[fromOffset: contentBase, size: contentSize] + precondition(slice.starts(with: "pbzx".utf8)) + return UnsafeBufferPointer(rebasing: slice) + } + + static func main() async throws { + let handle = try FileHandle(forReadingFrom: options.input) + try handle.seekToEnd() + let length = Int(try handle.offset()) + let file = UnsafeBufferPointer(start: mmap(nil, length, PROT_READ, MAP_PRIVATE, handle.fileDescriptor, 0).bindMemory(to: UInt8.self, capacity: length), count: length) + precondition(UnsafeMutableRawPointer(mutating: file.baseAddress) != MAP_FAILED) + defer { + munmap(UnsafeMutableRawPointer(mutating: file.baseAddress), length) + } + + if let output = options.output { + guard chdir(output.path) == 0 else { + fputs("Failed to access output directory at \(output.path): \(String(cString: strerror(errno)))", stderr) + exit(EXIT_FAILURE) + } + } + + await parseContent(locateContent(in: file)) + } +} From 5a18a6ec073d350a06d0199840f88166dd556d78 Mon Sep 17 00:00:00 2001 From: JP Simard Date: Wed, 2 Mar 2022 21:18:53 -0500 Subject: [PATCH 2/5] Adjust Unxip.swift for use in xcodes --- Sources/Unxip/Unxip.swift | 100 +++++++++----------------------------- 1 file changed, 22 insertions(+), 78 deletions(-) diff --git a/Sources/Unxip/Unxip.swift b/Sources/Unxip/Unxip.swift index 019e19a..7f3b9d5 100644 --- a/Sources/Unxip/Unxip.swift +++ b/Sources/Unxip/Unxip.swift @@ -1,6 +1,9 @@ import Compression import Foundation +// From: https://github.com/saagarjha/unxip +// License: GNU Lesser General Public License v3.0 + extension RandomAccessCollection { subscript(fromOffset fromOffset: Int = 0, toOffset toOffset: Int? = nil) -> SubSequence { let toOffset = toOffset ?? count @@ -13,6 +16,7 @@ extension RandomAccessCollection { } } +@available(macOS 11, *) extension AsyncStream.Continuation { func yieldWithBackoff(_ value: Element) async { let backoff: UInt64 = 1_000_000 @@ -22,6 +26,7 @@ extension AsyncStream.Continuation { } } +@available(macOS 11, *) struct ConcurrentStream { let batchSize: Int var operations = [@Sendable () async throws -> TaskResult]() @@ -96,6 +101,7 @@ final class Chunk: Sendable { } } +@available(macOS 11, *) struct File { let dev: Int let ino: Int @@ -217,81 +223,25 @@ struct File { } } -extension option { - init(name: StaticString, has_arg: CInt, flag: UnsafeMutablePointer?, val: StringLiteralType) { - let _option = name.withUTF8Buffer { - $0.withMemoryRebound(to: CChar.self) { - option(name: $0.baseAddress, has_arg: has_arg, flag: flag, val: CInt(UnicodeScalar(val)!.value)) - } - } - self = _option - } -} - -struct Options { +public struct UnxipOptions { var input: URL var output: URL? - var compress: Bool = true - - init() { - let options = [ - option(name: "compression-disable", has_arg: no_argument, flag: nil, val: "c"), - option(name: "help", has_arg: no_argument, flag: nil, val: "h"), - option(name: nil, has_arg: 0, flag: nil, val: 0), - ] - var result: CInt - repeat { - result = getopt_long(CommandLine.argc, CommandLine.unsafeArgv, "ch", options, nil) - if result < 0 { - break - } - switch UnicodeScalar(UInt32(result)) { - case "c": - compress = false - case "h": - Self.printUsage(nominally: true) - default: - Self.printUsage(nominally: false) - } - } while true - - let arguments = UnsafeBufferPointer(start: CommandLine.unsafeArgv + Int(optind), count: Int(CommandLine.argc - optind)).map { - String(cString: $0!) - } - guard let input = arguments.first else { - Self.printUsage(nominally: false) - } - - self.input = URL(fileURLWithPath: input) - - guard let output = arguments.dropFirst().first else { - return - } - - self.output = URL(fileURLWithPath: output) + public init(input: URL, output: URL) { + self.input = input + self.output = output } +} - static func printUsage(nominally: Bool) -> Never { - fputs( - """ - A fast Xcode unarchiver - - USAGE: unxip [options] [output] +@available(macOS 11, *) +public struct Unxip { + let options: UnxipOptions - OPTIONS: - -c, --compression-disable Disable APFS compression of result. - -h, --help Print this help message. - """, nominally ? stdout : stderr) - exit(nominally ? EXIT_SUCCESS : EXIT_FAILURE) + public init(options: UnxipOptions) { + self.options = options } -} - -@main -struct Main { - static let options = Options() - static func read(_ type: Integer.Type, from buffer: inout Buffer) -> Integer where Buffer.Element == UInt8, Buffer.SubSequence == Buffer { + func read(_ type: Integer.Type, from buffer: inout Buffer) -> Integer where Buffer.Element == UInt8, Buffer.SubSequence == Buffer { defer { buffer = buffer[fromOffset: MemoryLayout.size] } @@ -304,7 +254,7 @@ struct Main { return result } - static func chunks(from content: UnsafeBufferPointer) -> ConcurrentStream { + func chunks(from content: UnsafeBufferPointer) -> ConcurrentStream { var remaining = content[fromOffset: 4] let chunkSize = read(UInt64.self, from: &remaining) var decompressedSize: UInt64 = 0 @@ -337,7 +287,7 @@ struct Main { return chunkStream } - static func files(in chunkStream: ChunkStream) -> AsyncStream where ChunkStream.Element == Chunk { + func files(in chunkStream: ChunkStream) -> AsyncStream where ChunkStream.Element == Chunk { AsyncStream(bufferingPolicy: .bufferingOldest(ProcessInfo.processInfo.activeProcessorCount)) { continuation in Task { var iterator = chunkStream.makeAsyncIterator() @@ -401,7 +351,7 @@ struct Main { } } - static func parseContent(_ content: UnsafeBufferPointer) async { + func parseContent(_ content: UnsafeBufferPointer) async { var taskStream = ConcurrentStream(batchSize: 64) // Worst case, should allow for files up to 64 * 16MB = 1GB var hardlinks = [File.Identifier: (String, Task)]() var directories = [Substring: Task]() @@ -478,12 +428,6 @@ struct Main { setStickyBit(on: file) } - if options.compress, - await file.writeCompressedIfPossible(usingDescriptor: fd) - { - return - } - // pwritev requires the vector count to be positive if file.data.count == 0 { return @@ -515,7 +459,7 @@ struct Main { } } - static func locateContent(in file: UnsafeBufferPointer) -> UnsafeBufferPointer { + func locateContent(in file: UnsafeBufferPointer) -> UnsafeBufferPointer { precondition(file.starts(with: "xar!".utf8)) // magic var header = file[4...] let headerSize = read(UInt16.self, from: &header) @@ -543,7 +487,7 @@ struct Main { return UnsafeBufferPointer(rebasing: slice) } - static func main() async throws { + public func run() async throws { let handle = try FileHandle(forReadingFrom: options.input) try handle.seekToEnd() let length = Int(try handle.offset()) From 81758afc9d5b6b916dd04c340456dc38104d3dba Mon Sep 17 00:00:00 2001 From: JP Simard Date: Wed, 2 Mar 2022 17:05:21 -0500 Subject: [PATCH 3/5] Hook up experimental unxip flag --- Package.swift | 1 + Sources/XcodesKit/XcodeInstaller.swift | 39 +++++++++++++++++++------- Sources/xcodes/main.swift | 5 +++- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/Package.swift b/Package.swift index cdc0ba6..4a466f7 100644 --- a/Package.swift +++ b/Package.swift @@ -45,6 +45,7 @@ let package = Package( "PromiseKit", "PMKFoundation", "SwiftSoup", + "Unxip", "Version", .product(name: "XCModel", package: "XcodeReleases"), "Rainbow", diff --git a/Sources/XcodesKit/XcodeInstaller.swift b/Sources/XcodesKit/XcodeInstaller.swift index bbd872e..fc3171d 100644 --- a/Sources/XcodesKit/XcodeInstaller.swift +++ b/Sources/XcodesKit/XcodeInstaller.swift @@ -5,6 +5,7 @@ import AppleAPI import Version import LegibleError import Rainbow +import Unxip /// Downloads and installs Xcodes public final class XcodeInstaller { @@ -155,9 +156,9 @@ public final class XcodeInstaller { case aria2(Path) } - public func install(_ installationType: InstallationType, dataSource: DataSource, downloader: Downloader, destination: Path) -> Promise { + public func install(_ installationType: InstallationType, dataSource: DataSource, downloader: Downloader, destination: Path, experimentalUnxip: Bool = false) -> Promise { return firstly { () -> Promise in - return self.install(installationType, dataSource: dataSource, downloader: downloader, destination: destination, attemptNumber: 0) + return self.install(installationType, dataSource: dataSource, downloader: downloader, destination: destination, attemptNumber: 0, experimentalUnxip: experimentalUnxip) } .done { xcode in Current.logging.log("\nXcode \(xcode.version.descriptionWithoutBuildMetadata) has been installed to \(xcode.path.string)".green) @@ -165,12 +166,12 @@ public final class XcodeInstaller { } } - private func install(_ installationType: InstallationType, dataSource: DataSource, downloader: Downloader, destination: Path, attemptNumber: Int) -> Promise { + private func install(_ installationType: InstallationType, dataSource: DataSource, downloader: Downloader, destination: Path, attemptNumber: Int, experimentalUnxip: Bool) -> Promise { return firstly { () -> Promise<(Xcode, URL)> in return self.getXcodeArchive(installationType, dataSource: dataSource, downloader: downloader, destination: destination, willInstall: true) } .then { xcode, url -> Promise in - return self.installArchivedXcode(xcode, at: url, to: destination) + return self.installArchivedXcode(xcode, at: url, to: destination, experimentalUnxip: experimentalUnxip) } .recover { error -> Promise in switch error { @@ -187,7 +188,7 @@ public final class XcodeInstaller { Current.logging.log(error.legibleLocalizedDescription.red) Current.logging.log("Removing damaged XIP and re-attempting installation.\n") try Current.files.removeItem(at: damagedXIPURL) - return self.install(installationType, dataSource: dataSource, downloader: downloader, destination: destination, attemptNumber: attemptNumber + 1) + return self.install(installationType, dataSource: dataSource, downloader: downloader, destination: destination, attemptNumber: attemptNumber + 1, experimentalUnxip: experimentalUnxip) } } default: @@ -520,7 +521,7 @@ public final class XcodeInstaller { } } - public func installArchivedXcode(_ xcode: Xcode, at archiveURL: URL, to destination: Path) -> Promise { + public func installArchivedXcode(_ xcode: Xcode, at archiveURL: URL, to destination: Path, experimentalUnxip: Bool = false) -> Promise { let passwordInput = { Promise { seal in Current.logging.log("xcodes requires superuser privileges in order to finish installation.") @@ -533,7 +534,7 @@ public final class XcodeInstaller { let destinationURL = destination.join("Xcode-\(xcode.version.descriptionWithoutBuildMetadata).app").url switch archiveURL.pathExtension { case "xip": - return unarchiveAndMoveXIP(at: archiveURL, to: destinationURL).map { xcodeURL in + return unarchiveAndMoveXIP(at: archiveURL, to: destinationURL, experimentalUnxip: experimentalUnxip).map { xcodeURL in guard let path = Path(url: xcodeURL), Current.files.fileExists(atPath: path.string), @@ -717,9 +718,26 @@ public final class XcodeInstaller { } } - func unarchiveAndMoveXIP(at source: URL, to destination: URL) -> Promise { - return firstly { () -> Promise in + func unarchiveAndMoveXIP(at source: URL, to destination: URL, experimentalUnxip: Bool) -> Promise { + return firstly { () -> Promise in Current.logging.log(InstallationStep.unarchiving.description) + + if experimentalUnxip, #available(macOS 11, *) { + return Promise { seal in + Task.detached { + let output = source.deletingLastPathComponent() + let options = UnxipOptions(input: source, output: output) + + do { + try await Unxip(options: options).run() + seal.resolve(.fulfilled(())) + } catch { + seal.reject(error) + } + } + } + } + return Current.shell.unxip(source) .recover { (error) throws -> Promise in if case Process.PMKError.execution(_, _, let standardError) = error, @@ -728,8 +746,9 @@ public final class XcodeInstaller { } throw error } + .map { _ in () } } - .map { output -> URL in + .map { _ -> URL in Current.logging.log(InstallationStep.moving(destination: destination.path).description) let xcodeURL = source.deletingLastPathComponent().appendingPathComponent("Xcode.app") diff --git a/Sources/xcodes/main.swift b/Sources/xcodes/main.swift index 5e37158..91f818e 100644 --- a/Sources/xcodes/main.swift +++ b/Sources/xcodes/main.swift @@ -183,6 +183,9 @@ struct Xcodes: ParsableCommand { @Flag(help: "Don't use aria2 to download Xcode, even if its available.") var noAria2: Bool = false + @Flag(help: "Use the experimental unxip functionality. May speed up unarchiving by up to 2-3x.") + var experimentalUnxip: Bool = false + @Option(help: "The directory to install Xcode into. Defaults to /Applications.", completion: .directory) var directory: String? @@ -218,7 +221,7 @@ struct Xcodes: ParsableCommand { let destination = getDirectory(possibleDirectory: directory) - installer.install(installation, dataSource: globalDataSource.dataSource, downloader: downloader, destination: destination) + installer.install(installation, dataSource: globalDataSource.dataSource, downloader: downloader, destination: destination, experimentalUnxip: experimentalUnxip) .done { Install.exit() } .catch { error in Install.processDownloadOrInstall(error: error) From 816130a2390d26c90f0444423d9320a0a14db67d Mon Sep 17 00:00:00 2001 From: JP Simard Date: Wed, 2 Mar 2022 22:07:53 -0500 Subject: [PATCH 4/5] Re-enable APFS compression of unxip output I removed this by mistake. --- Sources/Unxip/Unxip.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Sources/Unxip/Unxip.swift b/Sources/Unxip/Unxip.swift index 7f3b9d5..a078bd6 100644 --- a/Sources/Unxip/Unxip.swift +++ b/Sources/Unxip/Unxip.swift @@ -428,6 +428,10 @@ public struct Unxip { setStickyBit(on: file) } + if await file.writeCompressedIfPossible(usingDescriptor: fd) { + return + } + // pwritev requires the vector count to be positive if file.data.count == 0 { return From 0e679fd348bf49e8943efdd166e6856d1aff3d42 Mon Sep 17 00:00:00 2001 From: JP Simard Date: Tue, 8 Mar 2022 11:44:42 -0500 Subject: [PATCH 5/5] Log hints about experimental unxip flag when unarchiving Co-authored-by: Matt Kiazyk --- Sources/XcodesKit/XcodeInstaller.swift | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Sources/XcodesKit/XcodeInstaller.swift b/Sources/XcodesKit/XcodeInstaller.swift index fc3171d..81aba81 100644 --- a/Sources/XcodesKit/XcodeInstaller.swift +++ b/Sources/XcodesKit/XcodeInstaller.swift @@ -80,7 +80,7 @@ public final class XcodeInstaller { /// A numbered step enum InstallationStep: CustomStringConvertible { case downloading(version: String, progress: String?, willInstall: Bool) - case unarchiving + case unarchiving(experimentalUnxip: Bool) case moving(destination: String) case trashingArchive(archiveName: String) case checkingSecurity @@ -103,8 +103,15 @@ public final class XcodeInstaller { } else { return "Downloading Xcode \(version)" } - case .unarchiving: - return "Unarchiving Xcode (This can take a while)" + case .unarchiving(let experimentalUnxip): + let hint = experimentalUnxip ? + "Using experimental unxip. If you encounter any issues, remove the flag and try again" : + "Using regular unxip. Try passing `--experimental-unxip` for a faster unxip process" + return + """ + Unarchiving Xcode (This can take a while) + \(hint) + """ case .moving(let destination): return "Moving Xcode to \(destination)" case .trashingArchive(let archiveName): @@ -720,7 +727,7 @@ public final class XcodeInstaller { func unarchiveAndMoveXIP(at source: URL, to destination: URL, experimentalUnxip: Bool) -> Promise { return firstly { () -> Promise in - Current.logging.log(InstallationStep.unarchiving.description) + Current.logging.log(InstallationStep.unarchiving(experimentalUnxip: experimentalUnxip).description) if experimentalUnxip, #available(macOS 11, *) { return Promise { seal in