Merge main into release/6.2

CMakeLists.txt

@@ -25,6 +25,7 @@ find_package(ArgumentParser CONFIG REQUIRED)
 find_package(SwiftCollections QUIET)
 find_package(SwiftSyntax CONFIG REQUIRED)
 find_package(SwiftCrypto CONFIG REQUIRED)
+find_package(SwiftASN1 CONFIG REQUIRED)
 
 include(SwiftSupport)
 

Sources/BuildSystemIntegration/BuildTargetIdentifierExtensions.swift

@@ -99,7 +99,6 @@ extension BuildTargetIdentifier {
 // MARK: BuildTargetIdentifier for CompileCommands
 
 extension BuildTargetIdentifier {
-  /// - Important: *For testing only*
   package static func createCompileCommands(compiler: String) throws -> BuildTargetIdentifier {
     var components = URLComponents()
     components.scheme = "compilecommands"

Sources/BuildSystemIntegration/CMakeLists.txt

@@ -19,6 +19,7 @@ add_library(BuildSystemIntegration STATIC
   LegacyBuildServerBuildSystem.swift
   MainFilesProvider.swift
   SplitShellCommand.swift
+  SwiftlyResolver.swift
   SwiftPMBuildSystem.swift)
 set_target_properties(BuildSystemIntegration PROPERTIES
   INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY})
@@ -35,7 +36,8 @@ target_link_libraries(BuildSystemIntegration PUBLIC
   PackageModel
   TSCBasic
   Build
-  SourceKitLSPAPI)
+  SourceKitLSPAPI
+  SwiftASN1)
 
 target_link_libraries(BuildSystemIntegration PRIVATE
   SKUtilities

Sources/BuildSystemIntegration/JSONCompilationDatabaseBuildSystem.swift

@@ -25,8 +25,23 @@ fileprivate extension CompilationDatabaseCompileCommand {
   /// without specifying a path.
   ///
   /// The absence of a compiler means we have an empty command line, which should never happen.
-  var compiler: String? {
-    return commandLine.first
+  ///
+  /// If the compiler is a symlink to `swiftly`, it uses `swiftlyResolver` to find the corresponding executable in a
+  /// real toolchain and returns that executable.
+  func compiler(swiftlyResolver: SwiftlyResolver) async -> String? {
+    guard let compiler = commandLine.first else {
+      return nil
+    }
+    let swiftlyResolved = await orLog("Resolving swiftly") {
+      try await swiftlyResolver.resolve(
+        compiler: URL(fileURLWithPath: compiler),
+        workingDirectory: URL(fileURLWithPath: directory)
+      )?.filePath
+    }
+    if let swiftlyResolved {
+      return swiftlyResolved
+    }
+    return compiler
   }
 }
 
@@ -49,14 +64,17 @@ package actor JSONCompilationDatabaseBuildSystem: BuiltInBuildSystem {
 
   package let configPath: URL
 
+  private let swiftlyResolver = SwiftlyResolver()
+
   // Watch for all all changes to `compile_commands.json` and `compile_flags.txt` instead of just the one at
   // `configPath` so that we cover the following semi-common scenario:
   // The user has a build that stores `compile_commands.json` in `mybuild`. In order to pick it  up, they create a
   // symlink from `<project root>/compile_commands.json` to `mybuild/compile_commands.json`.  We want to get notified
   // about the change to `mybuild/compile_commands.json` because it effectively changes the contents of
   // `<project root>/compile_commands.json`.
   package let fileWatchers: [FileSystemWatcher] = [
-    FileSystemWatcher(globPattern: "**/compile_commands.json", kind: [.create, .change, .delete])
+    FileSystemWatcher(globPattern: "**/compile_commands.json", kind: [.create, .change, .delete]),
+    FileSystemWatcher(globPattern: "**/.swift-version", kind: [.create, .change, .delete]),
   ]
 
   private var _indexStorePath: LazyValue<URL?> = .uninitialized
@@ -92,7 +110,11 @@ package actor JSONCompilationDatabaseBuildSystem: BuiltInBuildSystem {
   }
 
   package func buildTargets(request: WorkspaceBuildTargetsRequest) async throws -> WorkspaceBuildTargetsResponse {
-    let compilers = Set(compdb.commands.compactMap(\.compiler)).sorted { $0 < $1 }
+    let compilers = Set(
+      await compdb.commands.asyncCompactMap { (command) -> String? in
+        await command.compiler(swiftlyResolver: swiftlyResolver)
+      }
+    ).sorted { $0 < $1 }
     let targets = try await compilers.asyncMap { compiler in
       let toolchainUri: URI? =
         if let toolchainPath = await toolchainRegistry.toolchain(withCompiler: URL(fileURLWithPath: compiler))?.path {
@@ -115,12 +137,12 @@ package actor JSONCompilationDatabaseBuildSystem: BuiltInBuildSystem {
   }
 
   package func buildTargetSources(request: BuildTargetSourcesRequest) async throws -> BuildTargetSourcesResponse {
-    let items = request.targets.compactMap { (target) -> SourcesItem? in
+    let items = await request.targets.asyncCompactMap { (target) -> SourcesItem? in
       guard let targetCompiler = orLog("Compiler for target", { try target.compileCommandsCompiler }) else {
         return nil
       }
-      let commandsWithRequestedCompilers = compdb.commands.lazy.filter { command in
-        return targetCompiler == command.compiler
+      let commandsWithRequestedCompilers = await compdb.commands.lazy.asyncFilter { command in
+        return await targetCompiler == command.compiler(swiftlyResolver: swiftlyResolver)
       }
       let sources = commandsWithRequestedCompilers.map {
         SourceItem(uri: $0.uri, kind: .file, generated: false)
@@ -131,10 +153,14 @@ package actor JSONCompilationDatabaseBuildSystem: BuiltInBuildSystem {
     return BuildTargetSourcesResponse(items: items)
   }
 
-  package func didChangeWatchedFiles(notification: OnWatchedFilesDidChangeNotification) {
+  package func didChangeWatchedFiles(notification: OnWatchedFilesDidChangeNotification) async {
     if notification.changes.contains(where: { $0.uri.fileURL?.lastPathComponent == Self.dbName }) {
       self.reloadCompilationDatabase()
     }
+    if notification.changes.contains(where: { $0.uri.fileURL?.lastPathComponent == ".swift-version" }) {
+      await swiftlyResolver.clearCache()
+      connectionToSourceKitLSP.send(OnBuildTargetDidChangeNotification(changes: nil))
+    }
   }
 
   package func prepare(request: BuildTargetPrepareRequest) async throws -> VoidResponse {
@@ -145,8 +171,8 @@ package actor JSONCompilationDatabaseBuildSystem: BuiltInBuildSystem {
     request: TextDocumentSourceKitOptionsRequest
   ) async throws -> TextDocumentSourceKitOptionsResponse? {
     let targetCompiler = try request.target.compileCommandsCompiler
-    let command = compdb[request.textDocument.uri].filter {
-      $0.compiler == targetCompiler
+    let command = await compdb[request.textDocument.uri].asyncFilter {
+      return await $0.compiler(swiftlyResolver: swiftlyResolver) == targetCompiler
     }.first
     guard let command else {
       return nil

Sources/BuildSystemIntegration/SwiftlyResolver.swift

@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Foundation
+import SKUtilities
+import SwiftExtensions
+import TSCExtensions
+
+import struct TSCBasic.AbsolutePath
+import class TSCBasic.Process
+
+/// Given a path to a compiler, which might be a symlink to `swiftly`, this type determines the compiler executable in
+/// an actual toolchain. It also caches the results. The client needs to invalidate the cache if the path that swiftly
+/// might resolve to has changed, eg. because `.swift-version` has been updated.
+actor SwiftlyResolver {
+  private struct CacheKey: Hashable {
+    let compiler: URL
+    let workingDirectory: URL?
+  }
+
+  private var cache: LRUCache<CacheKey, Result<URL?, Error>> = LRUCache(capacity: 100)
+
+  /// Check if `compiler` is a symlink to `swiftly`. If so, find the executable in the toolchain that swiftly resolves
+  /// to within the given working directory and return the URL of the corresponding compiler in that toolchain.
+  /// If `compiler` does not resolve to `swiftly`, return `nil`.
+  func resolve(compiler: URL, workingDirectory: URL?) async throws -> URL? {
+    let cacheKey = CacheKey(compiler: compiler, workingDirectory: workingDirectory)
+    if let cached = cache[cacheKey] {
+      return try cached.get()
+    }
+    let computed: Result<URL?, Error>
+    do {
+      computed = .success(
+        try await resolveSwiftlyTrampolineImpl(compiler: compiler, workingDirectory: workingDirectory)
+      )
+    } catch {
+      computed = .failure(error)
+    }
+    cache[cacheKey] = computed
+    return try computed.get()
+  }
+
+  private func resolveSwiftlyTrampolineImpl(compiler: URL, workingDirectory: URL?) async throws -> URL? {
+    let realpath = try compiler.realpath
+    guard realpath.lastPathComponent == "swiftly" else {
+      return nil
+    }
+    let swiftlyResult = try await Process.run(
+      arguments: [realpath.filePath, "use", "-p"],
+      workingDirectory: try AbsolutePath(validatingOrNil: workingDirectory?.filePath)
+    )
+    let swiftlyToolchain = URL(
+      fileURLWithPath: try swiftlyResult.utf8Output().trimmingCharacters(in: .whitespacesAndNewlines)
+    )
+    let resolvedCompiler = swiftlyToolchain.appending(components: "usr", "bin", compiler.lastPathComponent)
+    if FileManager.default.fileExists(at: resolvedCompiler) {
+      return resolvedCompiler
+    }
+    return nil
+  }
+
+  func clearCache() {
+    cache.removeAll()
+  }
+}

Sources/SKTestSupport/CreateBinary.swift

@@ -0,0 +1,38 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+package import Foundation
+import SwiftExtensions
+import ToolchainRegistry
+import XCTest
+
+import class TSCBasic.Process
+
+/// Compiles the given Swift source code into a binary at `executablePath`.
+package func createBinary(_ sourceCode: String, at executablePath: URL) async throws {
+  try await withTestScratchDir { scratchDir in
+    let sourceFile = scratchDir.appending(component: "source.swift")
+    try await sourceCode.writeWithRetry(to: sourceFile)
+
+    var compilerArguments = try [
+      sourceFile.filePath,
+      "-o",
+      executablePath.filePath,
+    ]
+    if let defaultSDKPath {
+      compilerArguments += ["-sdk", defaultSDKPath]
+    }
+    try await Process.checkNonZeroExit(
+      arguments: [unwrap(ToolchainRegistry.forTesting.default?.swiftc?.filePath)] + compilerArguments
+    )
+  }
+}

Sources/SourceKitD/SourceKitD.swift

@@ -193,52 +193,56 @@ package actor SourceKitD {
     let dlopenModes: DLOpenFlags = [.lazy, .local, .first]
     #endif
     let dlhandle = try dlopen(path.filePath, mode: dlopenModes)
-    do {
-      try self.init(
-        dlhandle: dlhandle,
-        path: path,
-        pluginPaths: pluginPaths,
-        initialize: initialize
-      )
-    } catch {
-      try? dlhandle.close()
-      throw error
-    }
+    try self.init(
+      dlhandle: dlhandle,
+      path: path,
+      pluginPaths: pluginPaths,
+      initialize: initialize
+    )
   }
 
+  /// Create a `SourceKitD` instance from an existing `DLHandle`. `SourceKitD` takes over ownership of the `DLHandler`
+  /// and will close it when the `SourceKitD` instance gets deinitialized or if the initializer throws.
   package init(dlhandle: DLHandle, path: URL, pluginPaths: PluginPaths?, initialize: Bool) throws {
-    self.path = path
-    self.dylib = dlhandle
-    let api = try sourcekitd_api_functions_t(dlhandle)
-    self.api = api
-
-    // We load the plugin-related functions eagerly so the members are initialized and we don't have data races on first
-    // access to eg. `pluginApi`. But if one of the functions is missing, we will only emit that error when that family
-    // of functions is being used. For example, it is expected that the plugin functions are not available in
-    // SourceKit-LSP.
-    self.ideApiResult = Result(catching: { try sourcekitd_ide_api_functions_t(dlhandle) })
-    self.pluginApiResult = Result(catching: { try sourcekitd_plugin_api_functions_t(dlhandle) })
-    self.servicePluginApiResult = Result(catching: { try sourcekitd_service_plugin_api_functions_t(dlhandle) })
-
-    if let pluginPaths {
-      api.register_plugin_path?(pluginPaths.clientPlugin.path, pluginPaths.servicePlugin.path)
-    }
-    if initialize {
-      self.api.initialize()
-    }
+    do {
+      self.path = path
+      self.dylib = dlhandle
+      let api = try sourcekitd_api_functions_t(dlhandle)
+      self.api = api
+
+      // We load the plugin-related functions eagerly so the members are initialized and we don't have data races on first
+      // access to eg. `pluginApi`. But if one of the functions is missing, we will only emit that error when that family
+      // of functions is being used. For example, it is expected that the plugin functions are not available in
+      // SourceKit-LSP.
+      self.ideApiResult = Result(catching: { try sourcekitd_ide_api_functions_t(dlhandle) })
+      self.pluginApiResult = Result(catching: { try sourcekitd_plugin_api_functions_t(dlhandle) })
+      self.servicePluginApiResult = Result(catching: { try sourcekitd_service_plugin_api_functions_t(dlhandle) })
 
-    if initialize {
-      self.api.set_notification_handler { [weak self] rawResponse in
-        guard let self, let rawResponse else { return }
-        let response = SKDResponse(rawResponse, sourcekitd: self)
-        self.notificationHandlingQueue.async {
-          let handlers = await self.notificationHandlers.compactMap(\.value)
+      if let pluginPaths {
+        api.register_plugin_path?(pluginPaths.clientPlugin.path, pluginPaths.servicePlugin.path)
+      }
+      if initialize {
+        self.api.initialize()
+      }
+
+      if initialize {
+        self.api.set_notification_handler { [weak self] rawResponse in
+          guard let self, let rawResponse else { return }
+          let response = SKDResponse(rawResponse, sourcekitd: self)
+          self.notificationHandlingQueue.async {
+            let handlers = await self.notificationHandlers.compactMap(\.value)
 
-          for handler in handlers {
-            handler.notification(response)
+            for handler in handlers {
+              handler.notification(response)
+            }
           }
         }
       }
+    } catch {
+      orLog("Closing dlhandle after opening sourcekitd failed") {
+        try? dlhandle.close()
+      }
+      throw error
     }
   }
 

Sources/SourceKitD/dlopen.swift

@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+import SKLogging
 import SwiftExtensions
 
 #if os(Windows)
@@ -45,7 +46,9 @@ package final class DLHandle: Sendable {
   }
 
   deinit {
-    precondition(rawValue.value == nil, "DLHandle must be closed or explicitly leaked before destroying")
+    if rawValue.value != nil {
+      logger.fault("DLHandle must be closed or explicitly leaked before destroying")
+    }
   }
 
   /// The handle must not be used anymore after calling `close`.