Resolve swiftly when referenced in compile commands

ahoppen · ahoppen · commit a26b0b7b3a44 · 2025-05-09T09:12:21.000+02:00
When the compiler in `compile_commands.json` references a `swift` executable that’s a symlink to `swiftly`, SourceKit-LSP got confused because the `swift` executable doesn’t reside in a real toolchain, causing us to not provide any semantic functionality. When we discover that the `swift` executable reference in compile commands references a `swiftly` executable, use `swiftly use -p` to resolve the binary in the real toolchain and continue operating based on that. Fixes #2128 rdar://150301344
diff --git a/Sources/BuildSystemIntegration/BuildTargetIdentifierExtensions.swift b/Sources/BuildSystemIntegration/BuildTargetIdentifierExtensions.swift
@@ -99,7 +99,6 @@ extension BuildTargetIdentifier {
 // MARK: BuildTargetIdentifier for CompileCommands
 
 extension BuildTargetIdentifier {
-  /// - Important: *For testing only*
   package static func createCompileCommands(compiler: String) throws -> BuildTargetIdentifier {
     var components = URLComponents()
     components.scheme = "compilecommands"
diff --git a/Sources/BuildSystemIntegration/CMakeLists.txt b/Sources/BuildSystemIntegration/CMakeLists.txt
@@ -19,6 +19,7 @@ add_library(BuildSystemIntegration STATIC
   LegacyBuildServerBuildSystem.swift
   MainFilesProvider.swift
   SplitShellCommand.swift
+  SwiftlyResolver.swift
   SwiftPMBuildSystem.swift)
 set_target_properties(BuildSystemIntegration PROPERTIES
   INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY})
diff --git a/Sources/BuildSystemIntegration/JSONCompilationDatabaseBuildSystem.swift b/Sources/BuildSystemIntegration/JSONCompilationDatabaseBuildSystem.swift
@@ -25,8 +25,23 @@ fileprivate extension CompilationDatabaseCompileCommand {
   /// without specifying a path.
   ///
   /// The absence of a compiler means we have an empty command line, which should never happen.
-  var compiler: String? {
-    return commandLine.first
+  ///
+  /// If the compiler is a symlink to `swiftly`, it uses `swiftlyResolver` to find the corresponding executable in a
+  /// real toolchain and returns that executable.
+  func compiler(swiftlyResolver: SwiftlyResolver) async -> String? {
+    guard let compiler = commandLine.first else {
+      return nil
+    }
+    let swiftlyResolved = await orLog("Resolving swiftly") {
+      try await swiftlyResolver.resolve(
+        compiler: URL(fileURLWithPath: compiler),
+        workingDirectory: URL(fileURLWithPath: directory)
+      )?.filePath
+    }
+    if let swiftlyResolved {
+      return swiftlyResolved
+    }
+    return compiler
   }
 }
 
@@ -49,14 +64,17 @@ package actor JSONCompilationDatabaseBuildSystem: BuiltInBuildSystem {
 
   package let configPath: URL
 
+  private let swiftlyResolver = SwiftlyResolver()
+
   // Watch for all all changes to `compile_commands.json` and `compile_flags.txt` instead of just the one at
   // `configPath` so that we cover the following semi-common scenario:
   // The user has a build that stores `compile_commands.json` in `mybuild`. In order to pick it  up, they create a
   // symlink from `<project root>/compile_commands.json` to `mybuild/compile_commands.json`.  We want to get notified
   // about the change to `mybuild/compile_commands.json` because it effectively changes the contents of
   // `<project root>/compile_commands.json`.
   package let fileWatchers: [FileSystemWatcher] = [
-    FileSystemWatcher(globPattern: "**/compile_commands.json", kind: [.create, .change, .delete])
+    FileSystemWatcher(globPattern: "**/compile_commands.json", kind: [.create, .change, .delete]),
+    FileSystemWatcher(globPattern: "**/.swift-version", kind: [.create, .change, .delete]),
   ]
 
   private var _indexStorePath: LazyValue<URL?> = .uninitialized
@@ -92,7 +110,11 @@ package actor JSONCompilationDatabaseBuildSystem: BuiltInBuildSystem {
   }
 
   package func buildTargets(request: WorkspaceBuildTargetsRequest) async throws -> WorkspaceBuildTargetsResponse {
-    let compilers = Set(compdb.commands.compactMap(\.compiler)).sorted { $0 < $1 }
+    let compilers = Set(
+      await compdb.commands.asyncCompactMap { (command) -> String? in
+        await command.compiler(swiftlyResolver: swiftlyResolver)
+      }
+    ).sorted { $0 < $1 }
     let targets = try await compilers.asyncMap { compiler in
       let toolchainUri: URI? =
         if let toolchainPath = await toolchainRegistry.toolchain(withCompiler: URL(fileURLWithPath: compiler))?.path {
@@ -115,12 +137,12 @@ package actor JSONCompilationDatabaseBuildSystem: BuiltInBuildSystem {
   }
 
   package func buildTargetSources(request: BuildTargetSourcesRequest) async throws -> BuildTargetSourcesResponse {
-    let items = request.targets.compactMap { (target) -> SourcesItem? in
+    let items = await request.targets.asyncCompactMap { (target) -> SourcesItem? in
       guard let targetCompiler = orLog("Compiler for target", { try target.compileCommandsCompiler }) else {
         return nil
       }
-      let commandsWithRequestedCompilers = compdb.commands.lazy.filter { command in
-        return targetCompiler == command.compiler
+      let commandsWithRequestedCompilers = await compdb.commands.lazy.asyncFilter { command in
+        return await targetCompiler == command.compiler(swiftlyResolver: swiftlyResolver)
       }
       let sources = commandsWithRequestedCompilers.map {
         SourceItem(uri: $0.uri, kind: .file, generated: false)
@@ -131,10 +153,14 @@ package actor JSONCompilationDatabaseBuildSystem: BuiltInBuildSystem {
     return BuildTargetSourcesResponse(items: items)
   }
 
-  package func didChangeWatchedFiles(notification: OnWatchedFilesDidChangeNotification) {
+  package func didChangeWatchedFiles(notification: OnWatchedFilesDidChangeNotification) async {
     if notification.changes.contains(where: { $0.uri.fileURL?.lastPathComponent == Self.dbName }) {
       self.reloadCompilationDatabase()
     }
+    if notification.changes.contains(where: { $0.uri.fileURL?.lastPathComponent == ".swift-version" }) {
+      await swiftlyResolver.clearCache()
+      connectionToSourceKitLSP.send(OnBuildTargetDidChangeNotification(changes: nil))
+    }
   }
 
   package func prepare(request: BuildTargetPrepareRequest) async throws -> VoidResponse {
@@ -145,8 +171,8 @@ package actor JSONCompilationDatabaseBuildSystem: BuiltInBuildSystem {
     request: TextDocumentSourceKitOptionsRequest
   ) async throws -> TextDocumentSourceKitOptionsResponse? {
     let targetCompiler = try request.target.compileCommandsCompiler
-    let command = compdb[request.textDocument.uri].filter {
-      $0.compiler == targetCompiler
+    let command = await compdb[request.textDocument.uri].asyncFilter {
+      return await $0.compiler(swiftlyResolver: swiftlyResolver) == targetCompiler
     }.first
     guard let command else {
       return nil
diff --git a/Sources/BuildSystemIntegration/SwiftlyResolver.swift b/Sources/BuildSystemIntegration/SwiftlyResolver.swift
@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Foundation
+import SKUtilities
+import SwiftExtensions
+import TSCExtensions
+
+import struct TSCBasic.AbsolutePath
+import class TSCBasic.Process
+
+/// Given a path to a compiler, which might be a symlink to `swiftly`, this type determines the compiler executable in
+/// an actual toolchain. It also caches the results. The client needs to invalidate the cache if the path that swiftly
+/// might resolve to has changed, eg. because `.swift-version` has been updated.
+actor SwiftlyResolver {
+  private struct CacheKey: Hashable {
+    let compiler: URL
+    let workingDirectory: URL?
+  }
+
+  private var cache: LRUCache<CacheKey, Result<URL?, Error>> = LRUCache(capacity: 100)
+
+  /// Check if `compiler` is a symlink to `swiftly`. If so, find the executable in the toolchain that swiftly resolves
+  /// to within the given working directory and return the URL of the corresponding compiler in that toolchain.
+  /// If `compiler` does not resolve to `swiftly`, return `nil`.
+  func resolve(compiler: URL, workingDirectory: URL?) async throws -> URL? {
+    let cacheKey = CacheKey(compiler: compiler, workingDirectory: workingDirectory)
+    if let cached = cache[cacheKey] {
+      return try cached.get()
+    }
+    let computed: Result<URL?, Error>
+    do {
+      computed = .success(
+        try await resolveSwiftlyTrampolineImpl(compiler: compiler, workingDirectory: workingDirectory)
+      )
+    } catch {
+      computed = .failure(error)
+    }
+    cache[cacheKey] = computed
+    return try computed.get()
+  }
+
+  private func resolveSwiftlyTrampolineImpl(compiler: URL, workingDirectory: URL?) async throws -> URL? {
+    let realpath = try compiler.realpath
+    guard realpath.lastPathComponent == "swiftly" else {
+      return nil
+    }
+    let swiftlyResult = try await Process.run(
+      arguments: [realpath.filePath, "use", "-p"],
+      workingDirectory: try AbsolutePath(validatingOrNil: workingDirectory?.filePath)
+    )
+    let swiftlyToolchain = URL(
+      fileURLWithPath: try swiftlyResult.utf8Output().trimmingCharacters(in: .whitespacesAndNewlines)
+    )
+    let resolvedCompiler = swiftlyToolchain.appending(components: "usr", "bin", compiler.lastPathComponent)
+    if FileManager.default.fileExists(at: resolvedCompiler) {
+      return resolvedCompiler
+    }
+    return nil
+  }
+
+  func clearCache() {
+    cache.removeAll()
+  }
+}
diff --git a/Sources/SKTestSupport/CreateBinary.swift b/Sources/SKTestSupport/CreateBinary.swift
@@ -0,0 +1,38 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+package import Foundation
+import SwiftExtensions
+import ToolchainRegistry
+import XCTest
+
+import class TSCBasic.Process
+
+/// Compiles the given Swift source code into a binary at `executablePath`.
+package func createBinary(_ sourceCode: String, at executablePath: URL) async throws {
+  try await withTestScratchDir { scratchDir in
+    let sourceFile = scratchDir.appending(component: "source.swift")
+    try await sourceCode.writeWithRetry(to: sourceFile)
+
+    var compilerArguments = try [
+      sourceFile.filePath,
+      "-o",
+      executablePath.filePath,
+    ]
+    if let defaultSDKPath {
+      compilerArguments += ["-sdk", defaultSDKPath]
+    }
+    try await Process.checkNonZeroExit(
+      arguments: [unwrap(ToolchainRegistry.forTesting.default?.swiftc?.filePath)] + compilerArguments
+    )
+  }
+}
diff --git a/Tests/SourceKitLSPTests/CompilationDatabaseTests.swift b/Tests/SourceKitLSPTests/CompilationDatabaseTests.swift
@@ -11,10 +11,11 @@
 //===----------------------------------------------------------------------===//
 
 import BuildSystemIntegration
-import Foundation
 import LanguageServerProtocol
 import SKTestSupport
+import SwiftExtensions
 import TSCBasic
+import TSCExtensions
 import ToolchainRegistry
 import XCTest
 
@@ -238,6 +239,121 @@ final class CompilationDatabaseTests: XCTestCase {
       assertContains(error.message, "No language service")
     }
   }
+
+  func testLookThroughSwiftly() async throws {
+    try await withTestScratchDir { scratchDirectory in
+      let defaultToolchain = try await unwrap(ToolchainRegistry.forTesting.default)
+
+      // We create a toolchain registry with the default toolchain, which is able to provide semantic functionality and
+      // a dummy toolchain that can't provide semantic functionality.
+      let fakeToolchainURL = scratchDirectory.appending(components: "fakeToolchain")
+      let fakeToolchain = Toolchain(
+        identifier: "fake",
+        displayName: "fake",
+        path: fakeToolchainURL,
+        clang: nil,
+        swift: fakeToolchainURL.appending(components: "usr", "bin", "swift"),
+        swiftc: fakeToolchainURL.appending(components: "usr", "bin", "swiftc"),
+        swiftFormat: nil,
+        clangd: nil,
+        sourcekitd: fakeToolchainURL.appending(components: "usr", "lib", "sourcekitd.framework", "sourcekitd"),
+        libIndexStore: nil
+      )
+      let toolchainRegistry = ToolchainRegistry(toolchains: [
+        try await unwrap(ToolchainRegistry.forTesting.default), fakeToolchain,
+      ])
+
+      // We need to create a file for the swift executable because `SwiftlyResolver` checks for its presence.
+      try FileManager.default.createDirectory(
+        at: XCTUnwrap(fakeToolchain.swift).deletingLastPathComponent(),
+        withIntermediateDirectories: true
+      )
+      try await "".writeWithRetry(to: XCTUnwrap(fakeToolchain.swift))
+
+      // Create a dummy swiftly executable that picks the default toolchain for all file unless `fakeToolchain` is in
+      // the source file's path.
+      let dummySwiftlyExecutableUrl = scratchDirectory.appendingPathComponent("swiftly")
+      let dummySwiftExecutableUrl = scratchDirectory.appendingPathComponent("swift")
+      try FileManager.default.createSymbolicLink(
+        at: dummySwiftExecutableUrl,
+        withDestinationURL: dummySwiftlyExecutableUrl
+      )
+      try await createBinary(
+        """
+        import Foundation
+
+        if FileManager.default.currentDirectoryPath.contains("fakeToolchain") {
+          print(#"\(fakeToolchain.path.filePath)"#)
+        } else {
+          print(#"\(defaultToolchain.path.filePath)"#)
+        }
+        """,
+        at: dummySwiftlyExecutableUrl
+      )
+
+      // Now create a project in which we have one file in a `realToolchain` directory for which our fake swiftly will
+      // pick the default toolchain and one in `fakeToolchain` for which swiftly will pick the fake toolchain. We should
+      // be able to get semantic functionality for the file in `realToolchain` but not for `fakeToolchain` because
+      // sourcekitd can't be launched for that toolchain (since it doesn't exist).
+      let dummySwiftExecutablePathForJSON = try dummySwiftExecutableUrl.filePath.replacing(#"\"#, with: #"\\"#)
+
+      let project = try await MultiFileTestProject(
+        files: [
+          "realToolchain/realToolchain.swift": """
+          #warning("Test warning")
+          """,
+          "fakeToolchain/fakeToolchain.swift": """
+          #warning("Test warning")
+          """,
+          "compile_commands.json": """
+          [
+            {
+              "directory": "$TEST_DIR_BACKSLASH_ESCAPED/realToolchain",
+              "arguments": [
+                "\(dummySwiftExecutablePathForJSON)",
+                "$TEST_DIR_BACKSLASH_ESCAPED/realToolchain/realToolchain.swift",
+                \(defaultSDKArgs)
+              ],
+              "file": "realToolchain.swift",
+              "output": "$TEST_DIR_BACKSLASH_ESCAPED/realToolchain/test.swift.o"
+            },
+            {
+              "directory": "$TEST_DIR_BACKSLASH_ESCAPED/fakeToolchain",
+              "arguments": [
+                "\(dummySwiftExecutablePathForJSON)",
+                "$TEST_DIR_BACKSLASH_ESCAPED/fakeToolchain/fakeToolchain.swift",
+                \(defaultSDKArgs)
+              ],
+              "file": "fakeToolchain.swift",
+              "output": "$TEST_DIR_BACKSLASH_ESCAPED/fakeToolchain/test.swift.o"
+            }
+          ]
+          """,
+        ],
+        toolchainRegistry: toolchainRegistry
+      )
+
+      let (forRealToolchainUri, _) = try project.openDocument("realToolchain.swift")
+      let diagnostics = try await project.testClient.send(
+        DocumentDiagnosticsRequest(textDocument: TextDocumentIdentifier(forRealToolchainUri))
+      )
+      XCTAssertEqual(diagnostics.fullReport?.items.map(\.message), ["Test warning"])
+
+      let (forDummyToolchainUri, _) = try project.openDocument("fakeToolchain.swift")
+      await assertThrowsError(
+        try await project.testClient.send(
+          DocumentDiagnosticsRequest(textDocument: TextDocumentIdentifier(forDummyToolchainUri))
+        )
+      ) { error in
+        guard let error = error as? ResponseError else {
+          XCTFail("Expected ResponseError, got \(error)")
+          return
+        }
+        // The actual error message here doesn't matter too much, we just need to check that we don't get diagnostics.
+        assertContains(error.message, "No language service")
+      }
+    }
+  }
 }
 
 fileprivate let defaultSDKArgs: String = {
diff --git a/Tests/SourceKitLSPTests/FormattingTests.swift b/Tests/SourceKitLSPTests/FormattingTests.swift
@@ -299,18 +299,13 @@ final class FormattingTests: XCTestCase {
     try await withTestScratchDir { scratchDir in
       let toolchain = try await unwrap(ToolchainRegistry.forTesting.default)
 
-      let crashingSwiftFilePath = scratchDir.appendingPathComponent("crashing-executable.swift")
       let crashingExecutablePath = scratchDir.appendingPathComponent("crashing-executable")
-      try await "fatalError()".writeWithRetry(to: crashingSwiftFilePath)
-      var compilerArguments = try [
-        crashingSwiftFilePath.filePath,
-        "-o",
-        crashingExecutablePath.filePath,
-      ]
-      if let defaultSDKPath {
-        compilerArguments += ["-sdk", defaultSDKPath]
-      }
-      try await Process.checkNonZeroExit(arguments: [XCTUnwrap(toolchain.swiftc?.filePath)] + compilerArguments)
+      try await createBinary(
+        """
+        fatalError()
+        """,
+        at: crashingExecutablePath
+      )
 
       let toolchainRegistry = ToolchainRegistry(toolchains: [
         Toolchain(
