diff --git a/Sources/SwiftBuildSupport/PIFBuilder.swift b/Sources/SwiftBuildSupport/PIFBuilder.swift index 6e304910499..ba3e212c5a8 100644 --- a/Sources/SwiftBuildSupport/PIFBuilder.swift +++ b/Sources/SwiftBuildSupport/PIFBuilder.swift @@ -202,8 +202,9 @@ public final class PIFBuilder { return accessibleToolsPerPlugin } - /// Constructs a `PIF.TopLevelObject` representing the package graph. - package func constructPIF(buildParameters: BuildParameters) async throws -> PIF.TopLevelObject { + package func makePIFBuilders( + buildParameters: BuildParameters + ) async throws -> [(ResolvedPackage, PackagePIFBuilder, any PackagePIFBuilder.BuildDelegate)] { let pluginScriptRunner = self.parameters.pluginScriptRunner let outputDir = self.parameters.pluginWorkingDirectory.appending("outputs") @@ -218,225 +219,237 @@ public final class PIFBuilder { hostTriple: try pluginScriptRunner.hostTriple ) - return try await memoize(to: &self.cachedPIF) { - guard let rootPackage = self.graph.rootPackages.only else { - if self.graph.rootPackages.isEmpty { - throw PIFGenerationError.rootPackageNotFound - } else { - throw PIFGenerationError.multipleRootPackagesFound - } - } - - let sortedPackages = self.graph.packages - .sorted { $0.manifest.displayName < $1.manifest.displayName } // TODO: use identity instead? - - var packagesAndProjects: [(ResolvedPackage, ProjectModel.Project)] = [] - - for package in sortedPackages { - var buildToolPluginResultsByTargetName: [String: [PackagePIFBuilder.BuildToolPluginInvocationResult]] = [:] - - for module in package.modules { - // Apply each build tool plugin used by the target in order, - // creating a list of results (one for each plugin usage). - var buildToolPluginResults: [BuildToolPluginInvocationResult] = [] - var buildCommands: [PackagePIFBuilder.CustomBuildCommand] = [] - var prebuildCommands: [BuildToolPluginInvocationResult.PrebuildCommand] = [] - - for plugin in module.pluginDependencies(satisfying: buildParameters.buildEnvironment) { - let pluginModule = plugin.underlying as! PluginModule - - // Determine the tools to which this plugin has access, and create a name-to-path mapping from tool - // names to the corresponding paths. Built tools are assumed to be in the build tools directory. - guard let accessibleTools = availablePluginTools[plugin.id] else { - throw InternalError("No tools found for plugin \(plugin.name)") - } + let sortedPackages = self.graph.packages + .sorted { $0.manifest.displayName < $1.manifest.displayName } // TODO: use identity instead? - // Assign a plugin working directory based on the package, target, and plugin. - let pluginOutputDir = outputDir.appending( - components: [ - package.identity.description, - module.name, - buildParameters.destination == .host ? "tools" : "destination", - plugin.name, - ] - ) + var packagesAndBuilders: [(ResolvedPackage, PackagePIFBuilder, any PackagePIFBuilder.BuildDelegate)] = [] - // Determine the set of directories under which plugins are allowed to write. - // We always include just the output directory, and for now there is no possibility - // of opting into others. - let writableDirectories = [outputDir] - - // Determine a set of further directories under which plugins are never allowed - // to write, even if they are covered by other rules (such as being able to write - // to the temporary directory). - let readOnlyDirectories = [package.path] - - // In tools version 6.0 and newer, we vend the list of files generated by previous plugins. - let pluginDerivedSources: Sources - let pluginDerivedResources: [Resource] - if package.manifest.toolsVersion >= .v6_0 { - // Set up dummy observability because we don't want to emit diagnostics for this before the actual - // build. - let observability = ObservabilitySystem { _, _ in } - // Compute the generated files based on all results we have computed so far. - (pluginDerivedSources, pluginDerivedResources) = ModulesGraph.computePluginGeneratedFiles( - target: module, - toolsVersion: package.manifest.toolsVersion, - additionalFileRules: self.parameters.additionalFileRules, - buildParameters: buildParameters, - buildToolPluginInvocationResults: buildToolPluginResults, - prebuildCommandResults: [], - observabilityScope: observability.topScope - ) - } else { - pluginDerivedSources = .init(paths: [], root: package.path) - pluginDerivedResources = [] - } + for package in sortedPackages { + var buildToolPluginResultsByTargetName: [String: [PackagePIFBuilder.BuildToolPluginInvocationResult]] = [:] - let result = try await pluginModule.invoke( - module: plugin, - action: .createBuildToolCommands( - package: package, - target: module, - pluginGeneratedSources: pluginDerivedSources.paths, - pluginGeneratedResources: pluginDerivedResources.map(\.path) - ), - buildEnvironment: buildParameters.buildEnvironment, - scriptRunner: pluginScriptRunner, - workingDirectory: package.path, - outputDirectory: pluginOutputDir, - toolSearchDirectories: [buildParameters.toolchain.swiftCompilerPath.parentDirectory], - accessibleTools: accessibleTools, - writableDirectories: writableDirectories, - readOnlyDirectories: readOnlyDirectories, - allowNetworkConnections: [], - pkgConfigDirectories: self.parameters.pkgConfigDirectories, - sdkRootPath: buildParameters.toolchain.sdkRootPath, - fileSystem: fileSystem, - modulesGraph: self.graph, - observabilityScope: observabilityScope - ) + for module in package.modules { + // Apply each build tool plugin used by the target in order, + // creating a list of results (one for each plugin usage). + var buildToolPluginResults: [BuildToolPluginInvocationResult] = [] + var buildCommands: [PackagePIFBuilder.CustomBuildCommand] = [] + var prebuildCommands: [BuildToolPluginInvocationResult.PrebuildCommand] = [] - buildToolPluginResults.append(result) + for plugin in module.pluginDependencies(satisfying: buildParameters.buildEnvironment) { + let pluginModule = plugin.underlying as! PluginModule - let diagnosticsEmitter = observabilityScope.makeDiagnosticsEmitter { - var metadata = ObservabilityMetadata() - metadata.moduleName = module.name - metadata.pluginName = result.plugin.name - return metadata - } - - for line in result.textOutput.split(whereSeparator: { $0.isNewline }) { - diagnosticsEmitter.emit(info: line) - } + // Determine the tools to which this plugin has access, and create a name-to-path mapping from tool + // names to the corresponding paths. Built tools are assumed to be in the build tools directory. + guard let accessibleTools = availablePluginTools[plugin.id] else { + throw InternalError("No tools found for plugin \(plugin.name)") + } - for diag in result.diagnostics { - diagnosticsEmitter.emit(diag) - } + // Assign a plugin working directory based on the package, target, and plugin. + let pluginOutputDir = outputDir.appending( + components: [ + package.identity.description, + module.name, + buildParameters.destination == .host ? "tools" : "destination", + plugin.name, + ] + ) - prebuildCommands.append(contentsOf: result.prebuildCommands) - - buildCommands.append(contentsOf: result.buildCommands.map( { buildCommand in - var newEnv: Environment = buildCommand.configuration.environment - - // FIXME: This is largely a workaround for improper rpath setup on Linux. It should be - // removed once the Swift Build backend switches to use swiftc as the linker driver - // for targets with Swift sources. For now, limit the scope to non-macOS, so that - // plugins do not inadvertently use the toolchain stdlib instead of the OS stdlib - // when built with a Swift.org toolchain. - #if !os(macOS) - let runtimeLibPaths = buildParameters.toolchain.runtimeLibraryPaths - - // Add paths to swift standard runtime libraries to the library path so that they can be found at runtime - for libPath in runtimeLibPaths { - newEnv.appendPath(key: .libraryPath, value: libPath.pathString) - } - #endif - - // Append the system path at the end so that necessary system tool paths can be found - if let pathValue = Environment.current[EnvironmentKey.path] { - newEnv.appendPath(key: .path, value: pathValue) - } - - let writableDirectories: [AbsolutePath] = [pluginOutputDir] - - return PackagePIFBuilder.CustomBuildCommand( - displayName: buildCommand.configuration.displayName, - executable: buildCommand.configuration.executable.pathString, - arguments: buildCommand.configuration.arguments, - environment: .init(newEnv), - workingDir: package.path, - inputPaths: buildCommand.inputFiles, - outputPaths: buildCommand.outputFiles.map(\.pathString), - sandboxProfile: - self.parameters.disableSandbox ? - nil : - .init( - strictness: .writableTemporaryDirectory, - writableDirectories: writableDirectories, - readOnlyDirectories: buildCommand.inputFiles - ) - ) - })) + // Determine the set of directories under which plugins are allowed to write. + // We always include just the output directory, and for now there is no possibility + // of opting into others. + let writableDirectories = [outputDir] + + // Determine a set of further directories under which plugins are never allowed + // to write, even if they are covered by other rules (such as being able to write + // to the temporary directory). + let readOnlyDirectories = [package.path] + + // In tools version 6.0 and newer, we vend the list of files generated by previous plugins. + let pluginDerivedSources: Sources + let pluginDerivedResources: [Resource] + if package.manifest.toolsVersion >= .v6_0 { + // Set up dummy observability because we don't want to emit diagnostics for this before the actual + // build. + let observability = ObservabilitySystem { _, _ in } + // Compute the generated files based on all results we have computed so far. + (pluginDerivedSources, pluginDerivedResources) = ModulesGraph.computePluginGeneratedFiles( + target: module, + toolsVersion: package.manifest.toolsVersion, + additionalFileRules: self.parameters.additionalFileRules, + buildParameters: buildParameters, + buildToolPluginInvocationResults: buildToolPluginResults, + prebuildCommandResults: [], + observabilityScope: observability.topScope + ) + } else { + pluginDerivedSources = .init(paths: [], root: package.path) + pluginDerivedResources = [] } - // Run the prebuild commands generated from the plugin invocation now for this module. This will - // also give use the derived source code files needed for PIF generation. - let runResults = try Self.runPluginCommands( - using: self.pluginConfiguration, - for: buildToolPluginResults, + let result = try await pluginModule.invoke( + module: plugin, + action: .createBuildToolCommands( + package: package, + target: module, + pluginGeneratedSources: pluginDerivedSources.paths, + pluginGeneratedResources: pluginDerivedResources.map(\.path) + ), + buildEnvironment: buildParameters.buildEnvironment, + scriptRunner: pluginScriptRunner, + workingDirectory: package.path, + outputDirectory: pluginOutputDir, + toolSearchDirectories: [buildParameters.toolchain.swiftCompilerPath.parentDirectory], + accessibleTools: accessibleTools, + writableDirectories: writableDirectories, + readOnlyDirectories: readOnlyDirectories, + allowNetworkConnections: [], + pkgConfigDirectories: self.parameters.pkgConfigDirectories, + sdkRootPath: buildParameters.toolchain.sdkRootPath, fileSystem: fileSystem, + modulesGraph: self.graph, observabilityScope: observabilityScope ) - let result = PackagePIFBuilder.BuildToolPluginInvocationResult( - prebuildCommandOutputPaths: runResults.flatMap( { $0.derivedFiles }), - buildCommands: buildCommands - ) + buildToolPluginResults.append(result) - // Add a BuildToolPluginInvocationResult to the mapping. - if var existingResults = buildToolPluginResultsByTargetName[module.name] { - existingResults.append(result) - } else { - buildToolPluginResultsByTargetName[module.name] = [result] + let diagnosticsEmitter = observabilityScope.makeDiagnosticsEmitter { + var metadata = ObservabilityMetadata() + metadata.moduleName = module.name + metadata.pluginName = result.plugin.name + return metadata + } + + for line in result.textOutput.split(whereSeparator: { $0.isNewline }) { + diagnosticsEmitter.emit(info: line) + } + + for diag in result.diagnostics { + diagnosticsEmitter.emit(diag) } + + prebuildCommands.append(contentsOf: result.prebuildCommands) + + buildCommands.append(contentsOf: result.buildCommands.map( { buildCommand in + var newEnv: Environment = buildCommand.configuration.environment + + // FIXME: This is largely a workaround for improper rpath setup on Linux. It should be + // removed once the Swift Build backend switches to use swiftc as the linker driver + // for targets with Swift sources. For now, limit the scope to non-macOS, so that + // plugins do not inadvertently use the toolchain stdlib instead of the OS stdlib + // when built with a Swift.org toolchain. + #if !os(macOS) + let runtimeLibPaths = buildParameters.toolchain.runtimeLibraryPaths + + // Add paths to swift standard runtime libraries to the library path so that they can be found at runtime + for libPath in runtimeLibPaths { + newEnv.appendPath(key: .libraryPath, value: libPath.pathString) + } + #endif + + // Append the system path at the end so that necessary system tool paths can be found + if let pathValue = Environment.current[EnvironmentKey.path] { + newEnv.appendPath(key: .path, value: pathValue) + } + + let writableDirectories: [AbsolutePath] = [pluginOutputDir] + + return PackagePIFBuilder.CustomBuildCommand( + displayName: buildCommand.configuration.displayName, + executable: buildCommand.configuration.executable.pathString, + arguments: buildCommand.configuration.arguments, + environment: .init(newEnv), + workingDir: package.path, + inputPaths: buildCommand.inputFiles, + outputPaths: buildCommand.outputFiles.map(\.pathString), + sandboxProfile: + self.parameters.disableSandbox ? + nil : + .init( + strictness: .writableTemporaryDirectory, + writableDirectories: writableDirectories, + readOnlyDirectories: buildCommand.inputFiles + ) + ) + })) } - let packagePIFBuilderDelegate = PackagePIFBuilderDelegate( - package: package + // Run the prebuild commands generated from the plugin invocation now for this module. This will + // also give use the derived source code files needed for PIF generation. + let runResults = try Self.runPluginCommands( + using: self.pluginConfiguration, + for: buildToolPluginResults, + fileSystem: fileSystem, + observabilityScope: observabilityScope ) - let packagePIFBuilder = PackagePIFBuilder( - modulesGraph: self.graph, - resolvedPackage: package, - packageManifest: package.manifest, - delegate: packagePIFBuilderDelegate, - buildToolPluginResultsByTargetName: buildToolPluginResultsByTargetName, - createDylibForDynamicProducts: self.parameters.shouldCreateDylibForDynamicProducts, - addLocalRpaths: self.parameters.addLocalRpaths, - packageDisplayVersion: package.manifest.displayName, - fileSystem: self.fileSystem, - observabilityScope: self.observabilityScope + + let result = PackagePIFBuilder.BuildToolPluginInvocationResult( + prebuildCommandOutputPaths: runResults.flatMap( { $0.derivedFiles }), + buildCommands: buildCommands ) - - try packagePIFBuilder.build() - packagesAndProjects.append((package, packagePIFBuilder.pifProject)) + + // Add a BuildToolPluginInvocationResult to the mapping. + if var existingResults = buildToolPluginResultsByTargetName[module.name] { + existingResults.append(result) + } else { + buildToolPluginResultsByTargetName[module.name] = [result] + } } - - var projects = packagesAndProjects.map(\.1) - projects.append( - try buildAggregateProject( - packagesAndProjects: packagesAndProjects, + + let packagePIFBuilderDelegate = PackagePIFBuilderDelegate( + package: package + ) + let packagePIFBuilder = PackagePIFBuilder( + modulesGraph: self.graph, + resolvedPackage: package, + packageManifest: package.manifest, + delegate: packagePIFBuilderDelegate, + buildToolPluginResultsByTargetName: buildToolPluginResultsByTargetName, + createDylibForDynamicProducts: self.parameters.shouldCreateDylibForDynamicProducts, + addLocalRpaths: self.parameters.addLocalRpaths, + packageDisplayVersion: package.manifest.displayName, + fileSystem: self.fileSystem, + observabilityScope: self.observabilityScope + ) + + packagesAndBuilders.append((package, packagePIFBuilder, packagePIFBuilderDelegate)) + } + + return packagesAndBuilders + } + + /// Constructs a `PIF.TopLevelObject` representing the package graph. + package func constructPIF(buildParameters: BuildParameters) async throws -> PIF.TopLevelObject { + return try await memoize(to: &self.cachedPIF) { + let packagesAndPIFBuilders = try await makePIFBuilders(buildParameters: buildParameters) + + let packagesAndPIFProjects = try packagesAndPIFBuilders.map { (package, pifBuilder, _) in + try pifBuilder.build() + let pifProject: ProjectModel.Project = pifBuilder.pifProject + return (package, pifProject) + } + + var pifProjects: [ProjectModel.Project] = packagesAndPIFProjects.map(\.1) + pifProjects.append( + try buildAggregatePIFProject( + packagesAndProjects: packagesAndPIFProjects, observabilityScope: observabilityScope, modulesGraph: graph, buildParameters: buildParameters ) ) + guard let rootPackage = self.graph.rootPackages.only else { + if self.graph.rootPackages.isEmpty { + throw PIFGenerationError.rootPackageNotFound + } else { + throw PIFGenerationError.multipleRootPackagesFound + } + } + let workspace = PIF.Workspace( id: "Workspace:\(rootPackage.path.pathString)", name: rootPackage.manifest.displayName, // TODO: use identity instead? path: rootPackage.path, - projects: projects + projects: pifProjects ) return PIF.TopLevelObject(workspace: workspace) @@ -625,7 +638,7 @@ fileprivate final class PackagePIFBuilderDelegate: PackagePIFBuilder.BuildDelega } } -fileprivate func buildAggregateProject( +fileprivate func buildAggregatePIFProject( packagesAndProjects: [(package: ResolvedPackage, project: ProjectModel.Project)], observabilityScope: ObservabilityScope, modulesGraph: ModulesGraph, diff --git a/Sources/SwiftBuildSupport/PackagePIFBuilder.swift b/Sources/SwiftBuildSupport/PackagePIFBuilder.swift index 53c7a428be8..cf398c2ddfa 100644 --- a/Sources/SwiftBuildSupport/PackagePIFBuilder.swift +++ b/Sources/SwiftBuildSupport/PackagePIFBuilder.swift @@ -26,6 +26,7 @@ import class PackageModel.Product import struct PackageModel.Platform import struct PackageModel.PlatformVersion import struct PackageModel.Resource +import struct PackageModel.PackageIdentity import enum PackageModel.ProductType import struct PackageGraph.ModulesGraph @@ -347,8 +348,8 @@ public final class PackagePIFBuilder { public struct LinkedPackageBinary { public let name: String - public let packageName: String public let type: BinaryType + public let packageIdentity: PackageIdentity @frozen public enum BinaryType { @@ -356,10 +357,16 @@ public final class PackagePIFBuilder { case target } - public init(name: String, packageName: String, type: BinaryType) { - self.name = name - self.packageName = packageName - self.type = type + public init(product: String, packageIdentity: PackageIdentity) { + self.name = product + self.type = .product + self.packageIdentity = packageIdentity + } + + public init(module: String, packageIdentity: PackageIdentity) { + self.name = module + self.type = .target + self.packageIdentity = packageIdentity } } @@ -695,29 +702,27 @@ enum PIFBuildingError: Error { } extension PackagePIFBuilder.LinkedPackageBinary { - init?(module: ResolvedModule, package: ResolvedPackage) { - let packageName = package.manifest.displayName - + init?(module: ResolvedModule) { switch module.type { case .executable, .snippet, .test: - self.init(name: module.name, packageName: packageName, type: .product) + self.init(product: module.name, packageIdentity: module.packageIdentity) case .library, .binary, .macro: - self.init(name: module.name, packageName: packageName, type: .target) + self.init(module: module.name, packageIdentity: module.packageIdentity) case .systemModule, .plugin: return nil } } - init?(dependency: ResolvedModule.Dependency, package: ResolvedPackage) { + init?(dependency: ResolvedModule.Dependency) { switch dependency { case .product(let productDependency, _): guard productDependency.hasSourceTargets else { return nil } - self.init(name: productDependency.name, packageName: package.name, type: .product) + self.init(product: productDependency.name, packageIdentity: productDependency.packageIdentity) case .module(let moduleDependency, _): - self.init(module: moduleDependency, package: package) + self.init(module: moduleDependency) } } } diff --git a/Sources/SwiftBuildSupport/PackagePIFProjectBuilder+Modules.swift b/Sources/SwiftBuildSupport/PackagePIFProjectBuilder+Modules.swift index a36ea1b48f2..acde62e2a2c 100644 --- a/Sources/SwiftBuildSupport/PackagePIFProjectBuilder+Modules.swift +++ b/Sources/SwiftBuildSupport/PackagePIFProjectBuilder+Modules.swift @@ -785,7 +785,7 @@ extension PackagePIFProjectBuilder { // Collect linked binaries. let linkedPackageBinaries: [PackagePIFBuilder.LinkedPackageBinary] = sourceModule.dependencies.compactMap { - PackagePIFBuilder.LinkedPackageBinary(dependency: $0, package: self.package) + PackagePIFBuilder.LinkedPackageBinary(dependency: $0) } let productOrModuleType: PackagePIFBuilder.ModuleOrProductType = if desiredModuleType == .dynamicLibrary { diff --git a/Sources/SwiftBuildSupport/PackagePIFProjectBuilder+Products.swift b/Sources/SwiftBuildSupport/PackagePIFProjectBuilder+Products.swift index e7c75b928d6..ba35884771e 100644 --- a/Sources/SwiftBuildSupport/PackagePIFProjectBuilder+Products.swift +++ b/Sources/SwiftBuildSupport/PackagePIFProjectBuilder+Products.swift @@ -490,7 +490,7 @@ extension PackagePIFProjectBuilder { // Collect linked binaries. let linkedPackageBinaries: [PackagePIFBuilder.LinkedPackageBinary] = mainModule.dependencies.compactMap { - PackagePIFBuilder.LinkedPackageBinary(dependency: $0, package: self.package) + PackagePIFBuilder.LinkedPackageBinary(dependency: $0) } let moduleOrProduct = PackagePIFBuilder.ModuleOrProduct( @@ -844,7 +844,7 @@ extension PackagePIFProjectBuilder { // Collect linked binaries. let linkedPackageBinaries = product.modules.compactMap { - PackagePIFBuilder.LinkedPackageBinary(module: $0, package: self.package) + PackagePIFBuilder.LinkedPackageBinary(module: $0) } let moduleOrProductType: PackagePIFBuilder.ModuleOrProductType = switch product.libraryType { diff --git a/Tests/SwiftBuildSupportTests/PIFBuilderTests.swift b/Tests/SwiftBuildSupportTests/PIFBuilderTests.swift index 80f00260ee5..74e62e96887 100644 --- a/Tests/SwiftBuildSupportTests/PIFBuilderTests.swift +++ b/Tests/SwiftBuildSupportTests/PIFBuilderTests.swift @@ -43,7 +43,11 @@ extension PIFBuilderParameters { } } -fileprivate func withGeneratedPIF(fromFixture fixtureName: String, addLocalRpaths: Bool = true, do doIt: (SwiftBuildSupport.PIF.TopLevelObject, TestingObservability) async throws -> ()) async throws { +fileprivate func withGeneratedPIF( + fromFixture fixtureName: String, + addLocalRpaths: Bool = true, + do doIt: (SwiftBuildSupport.PIF.TopLevelObject, TestingObservability) async throws -> () +) async throws { try await fixture(name: fixtureName) { fixturePath in let observabilitySystem = ObservabilitySystem.makeForTesting() let workspace = try Workspace( @@ -59,7 +63,10 @@ fileprivate func withGeneratedPIF(fromFixture fixtureName: String, addLocalRpath ) let builder = PIFBuilder( graph: graph, - parameters: try PIFBuilderParameters.constructDefaultParametersForTesting(temporaryDirectory: fixturePath, addLocalRpaths: addLocalRpaths), + parameters: try PIFBuilderParameters.constructDefaultParametersForTesting( + temporaryDirectory: fixturePath, + addLocalRpaths: addLocalRpaths + ), fileSystem: localFileSystem, observabilityScope: observabilitySystem.topScope )