Change how libraries are specified to the linker when using searched libs

- remove the platform specifics from computeLibraryArgs (we cannot assume
  that all libraries have a lib prefix and what there suffix is.) So we
  now use the FileType prefix and remove any suffix when using
  searchPathFlagsForLD, moving this into the LinkerSpec.LibrarySpecifier
  extension, this allows for proper searching of libraries, and linking
  of dynamic libraries (especially on Windows).

Author: daveinglis

Sources/SWBCore/SpecImplementations/LinkerSpec.swift

@@ -89,7 +89,9 @@ open class LinkerSpec : CommandLineToolSpec, @unchecked Sendable {
         /// The path to the privacy file, if one exists.
         public let privacyFile: Path?
 
-        public init(kind: Kind, path: Path, mode: Mode, useSearchPaths: Bool, swiftModulePaths: [String: Path], swiftModuleAdditionalLinkerArgResponseFilePaths: [String: Path], explicitDependencies: [Path] = [], topLevelItemPath: Path? = nil, dsymPath: Path? = nil, xcframeworkSourcePath: Path? = nil, privacyFile: Path? = nil) {
+        public let libPrefix: String?
+
+        public init(kind: Kind, path: Path, mode: Mode, useSearchPaths: Bool, swiftModulePaths: [String: Path], swiftModuleAdditionalLinkerArgResponseFilePaths: [String: Path], prefixes: [String] = [], explicitDependencies: [Path] = [], topLevelItemPath: Path? = nil, dsymPath: Path? = nil, xcframeworkSourcePath: Path? = nil, privacyFile: Path? = nil) {
             self.kind = kind
             self.path = path
             self.mode = mode
@@ -101,6 +103,7 @@ open class LinkerSpec : CommandLineToolSpec, @unchecked Sendable {
             self.dsymPath = dsymPath
             self.xcframeworkSourcePath = xcframeworkSourcePath
             self.privacyFile = privacyFile
+            self.libPrefix = prefixes.first
         }
     }
 

Sources/SWBCore/SpecImplementations/Specs.swift

@@ -301,6 +301,9 @@ public class FileTypeSpec : Spec, SpecType, @unchecked Sendable {
     /// Returns `true` if the `isWrapperFolder` value is set in the XCSpec for the file spec.
     public let isWrapper: Bool
 
+    /// Returns any common prefix this file may have (currently used when specifying searched libraries to linker)
+    public let prefixes: [String]
+
     required init(_ parser: SpecParser, _ basedOnSpec: Spec?) {
         let basedOnFileTypeSpec = basedOnSpec as? FileTypeSpec ?? nil
 
@@ -318,8 +321,8 @@ public class FileTypeSpec : Spec, SpecType, @unchecked Sendable {
         self.isEmbeddableInProduct = parser.parseBool("IsEmbeddable") ?? false
         self.validateOnCopy = parser.parseBool("ValidateOnCopy") ?? false
         self.codeSignOnCopy = parser.parseBool("CodeSignOnCopy") ?? false
-
         self.isWrapper = parser.parseBool("IsWrapperFolder") ?? false
+        self.prefixes = parser.parseStringList("Prefix") ?? []
 
         // Parse and ignore keys we have no use for.
         //
@@ -358,7 +361,6 @@ public class FileTypeSpec : Spec, SpecType, @unchecked Sendable {
         parser.parseStringList("MIMETypes")
         parser.parseString("Permissions")
         parser.parseString("PlistStructureDefinition")
-        parser.parseStringList("Prefix")
         parser.parseBool("RemoveHeadersOnCopy")
         parser.parseBool("RequiresHardTabs")
         parser.parseString("UTI")

Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift

@@ -466,7 +466,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec
         }
 
         // Add the library arguments.
-        let libraryArgs = LdLinkerSpec.computeLibraryArgs(libraries, scope: cbc.scope)
+        let libraryArgs = LdLinkerSpec.computeLibraryArgs(libraries, scope: cbc.scope, delegate: delegate)
         specialArgs += libraryArgs.args
         if SWBFeatureFlag.enableLinkerInputsFromLibrarySpecifiers.value {
             inputPaths += libraryArgs.inputs
@@ -888,7 +888,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec
         var inputPaths = cbc.inputs.map({ $0.absolutePath })
 
         // Add the library arguments.
-        let libraryArgs = LdLinkerSpec.computeLibraryArgs(libraries, scope: cbc.scope)
+        let libraryArgs = LdLinkerSpec.computeLibraryArgs(libraries, scope: cbc.scope, delegate: delegate)
         specialArgs += libraryArgs.args
         if SWBFeatureFlag.enableLinkerInputsFromLibrarySpecifiers.value {
             inputPaths += libraryArgs.inputs
@@ -1286,44 +1286,18 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec
     /// Compute the list of command line arguments and inputs to pass to the linker, given a list of library specifiers.
     ///
     /// Note that `inputs` will only contain values for libraries which are being directly linked by absolute path rather than by using search paths.
-    private static func computeLibraryArgs(_ libraries: [LibrarySpecifier], scope: MacroEvaluationScope) -> (args: [String], inputs: [Path]) {
+    private static func computeLibraryArgs(_ libraries: [LibrarySpecifier], scope: MacroEvaluationScope, delegate: any TaskGenerationDelegate) -> (args: [String], inputs: [Path]) {
         // Construct the library arguments.
         return libraries.compactMap { specifier -> (args: [String], inputs: [Path]) in
-            let basename = specifier.path.basename
-
-            // FIXME: This isn't a good system, we need to redesign how we talk to the linker w.r.t. search paths and our notion of paths.
             switch specifier.kind {
-            case .static:
-                if specifier.useSearchPaths, basename.hasPrefix("lib"), basename.hasSuffix(".a") {
-                    return (specifier.searchPathFlagsForLd(basename.withoutPrefix("lib").withoutSuffix(".a")), [])
-                }
-                return (specifier.absolutePathFlagsForLd(), [specifier.path])
-            case .dynamic:
-                let suffix = ".\(scope.evaluate(BuiltinMacros.DYNAMIC_LIBRARY_EXTENSION))"
-                if specifier.useSearchPaths, basename.hasPrefix("lib"), basename.hasSuffix(suffix) {
-                    return (specifier.searchPathFlagsForLd(basename.withoutPrefix("lib").withoutSuffix(suffix)), [])
-                }
-                return (specifier.absolutePathFlagsForLd(), [specifier.path])
-            case .textBased:
-                if specifier.useSearchPaths, basename.hasPrefix("lib"), basename.hasSuffix(".tbd") {
-                    // .merge and .reexport are not supported for text-based libraries.
-                    return (specifier.searchPathFlagsForLd(basename.withoutPrefix("lib").withoutSuffix(".tbd")), [])
-                }
-                return (specifier.absolutePathFlagsForLd(), [specifier.path])
-            case .framework:
-                let frameworkName = Path(basename).withoutSuffix
+            case .static, .dynamic, .textBased, .framework:
                 if specifier.useSearchPaths {
-                    return (specifier.searchPathFlagsForLd(frameworkName), [])
-                }
-                let absPathArgs = specifier.absolutePathFlagsForLd()
-                let returnPath: Path
-                if let pathArg = absPathArgs.last, Path(pathArg).basename == frameworkName {
-                    returnPath = Path(pathArg)
-                }
-                else {
-                    returnPath = specifier.path
+                    let args = specifier.searchPathFlagsForLd()
+                    if !args.isEmpty { // no search args, fallback to absolute path to library
+                        return (args, [])
+                    }
                 }
-                return (absPathArgs, [returnPath])
+                return (specifier.absolutePathFlagsForLd(), [specifier.path])
             case .object:
                 // Object files are added to linker inputs in the sources task producer.
                 return ([], [])
@@ -1559,35 +1533,44 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec
 
 /// Extensions to `LinkerSpec.LibrarySpecifier` specific to the dynamic linker.
 fileprivate extension LinkerSpec.LibrarySpecifier {
-    func searchPathFlagsForLd(_ name: String) -> [String] {
+    func searchPathFlagsForLd() -> [String] {
+        let strippedName: String
+        if let prefix = libPrefix, path.basename.hasPrefix(prefix) {
+            strippedName = Path(path.basename).withoutSuffix.withoutPrefix(prefix)
+        } else {
+            if libPrefix != nil { // we need a prefix for linking with search paths
+                return [] // this will fallback to using absolute paths
+            }
+            strippedName = Path(path.basename).withoutSuffix
+        }
         switch (kind, mode) {
         case (.dynamic, .normal):
-            return ["-l" + name]
+            return ["-l" + strippedName]
         case (.dynamic, .reexport):
-            return ["-Xlinker", "-reexport-l" + name]
+            return ["-Xlinker", "-reexport-l" + strippedName]
         case (.dynamic, .merge):
-            return ["-Xlinker", "-merge-l" + name]
+            return ["-Xlinker", "-merge-l" + strippedName]
         case (.dynamic, .reexport_merge):
-            return ["-Xlinker", "-no_merge-l" + name]
+            return ["-Xlinker", "-no_merge-l" + strippedName]
         case (.dynamic, .weak):
-            return ["-weak-l" + name]
+            return ["-weak-l" + strippedName]
         case (.static, .weak),
              (.textBased, .weak):
-            return ["-weak-l" + name]
+            return ["-weak-l" + strippedName]
         case (.static, _),
              (.textBased, _):
             // Other modes are not supported for these kinds.
-            return ["-l" + name]
+            return ["-l" + strippedName]
         case (.framework, .normal):
-            return ["-framework", name]
+            return ["-framework", strippedName]
         case (.framework, .reexport):
-            return ["-Xlinker", "-reexport_framework", "-Xlinker", name]
+            return ["-Xlinker", "-reexport_framework", "-Xlinker", strippedName]
         case (.framework, .merge):
-            return ["-Xlinker", "-merge_framework", "-Xlinker", name]
+            return ["-Xlinker", "-merge_framework", "-Xlinker", strippedName]
         case (.framework, .reexport_merge):
-            return ["-Xlinker", "-no_merge_framework", "-Xlinker", name]
+            return ["-Xlinker", "-no_merge_framework", "-Xlinker", strippedName]
         case (.framework, .weak):
-            return ["-weak_framework", name]
+            return ["-weak_framework", strippedName]
         case (.object, _):
             // Object files are added to linker inputs in the sources task producer.
             return []
@@ -1724,15 +1707,17 @@ public final class LibtoolLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @u
                     delegate.warning("Product \(cbc.output.basename) cannot weak-link \(specifier.kind) \(basename)")
                 }
 
-                if specifier.useSearchPaths, basename.hasPrefix("lib"), basename.hasSuffix(".a") {
+                if specifier.useSearchPaths {
                     // Locate using search paths: Add a -l option and *don't* add the path to the library as an input to the task.
-                    return ["-l" + basename.withoutPrefix("lib").withoutSuffix(".a")]
-                }
-                else {
-                    // Locate using an absolute path: Add the path as an option and as an input to the task.
-                    inputPaths.append(specifier.path)
-                    return [specifier.path.str]
+                    let basename = specifier.path.basename
+                    let expectedPrefix = specifier.libPrefix ?? "lib"
+                    if basename.hasPrefix(expectedPrefix) {
+                        return ["-l" + Path(basename).withoutSuffix.withoutPrefix(expectedPrefix)]
+                    }
                 }
+                // Locate using an absolute path: Add the path as an option and as an input to the task.
+                inputPaths.append(specifier.path)
+                return [specifier.path.str]
 
             case .object:
                 // Object files are added to linker inputs in the sources task producer and so end up in the link-file-list.

Sources/SWBGenericUnixPlatform/Specs/Unix.xcspec

@@ -91,4 +91,14 @@
         IconNamePrefix = "TargetPlugin";
         DefaultTargetName = "Object File";
     },
+    {
+        Domain = generic-unix;
+        Type = FileType;
+        Identifier = compiled.mach-o.dylib;
+        BasedOn = compiled.mach-o;
+        Prefix = (lib);
+        Extensions = (so);
+        IsLibrary = YES;
+        IsDynamicLibrary = YES;
+    }
 )

Sources/SWBTaskConstruction/TaskProducers/BuildPhaseTaskProducers/SourcesTaskProducer.swift

@@ -507,6 +507,7 @@ package final class SourcesTaskProducer: FilesBasedBuildPhaseTaskProducerBase, F
                     useSearchPaths: useSearchPaths,
                     swiftModulePaths: swiftModulePaths,
                     swiftModuleAdditionalLinkerArgResponseFilePaths: swiftModuleAdditionalLinkerArgResponseFilePaths,
+                    prefixes: fileType.prefixes,
                     privacyFile: privacyFile
                 )
             } else if fileType.conformsTo(context.lookupFileType(identifier: "compiled.mach-o.dylib")!) {
@@ -517,6 +518,7 @@ package final class SourcesTaskProducer: FilesBasedBuildPhaseTaskProducerBase, F
                     useSearchPaths: useSearchPaths,
                     swiftModulePaths: [:],
                     swiftModuleAdditionalLinkerArgResponseFilePaths: [:],
+                    prefixes: fileType.prefixes,
                     privacyFile: privacyFile
                 )
             } else if fileType.conformsTo(context.lookupFileType(identifier: "sourcecode.text-based-dylib-definition")!) {
@@ -527,17 +529,18 @@ package final class SourcesTaskProducer: FilesBasedBuildPhaseTaskProducerBase, F
                     useSearchPaths: useSearchPaths,
                     swiftModulePaths: [:],
                     swiftModuleAdditionalLinkerArgResponseFilePaths: [:],
+                    prefixes: fileType.prefixes,
                     privacyFile: privacyFile
                 )
             } else if fileType.conformsTo(context.lookupFileType(identifier: "wrapper.framework")!) {
-                func kindFromSettings(_ settings: Settings) -> LinkerSpec.LibrarySpecifier.Kind? {
+                func kindFromSettings(_ settings: Settings) -> (kind: LinkerSpec.LibrarySpecifier.Kind, prefixes: [String])? {
                     switch settings.globalScope.evaluate(BuiltinMacros.MACH_O_TYPE) {
                     case "staticlib":
-                        return .static
+                        return (.static, context.lookupFileType(identifier: "archive.ar")?.prefixes ?? [])
                     case "mh_dylib":
-                        return .dynamic
+                        return (.dynamic, context.lookupFileType(identifier: "compiled.mach-o.dylib")?.prefixes ?? [])
                     case "mh_object":
-                        return .object
+                        return (.object, [])
                     default:
                         return nil
                     }
@@ -547,9 +550,11 @@ package final class SourcesTaskProducer: FilesBasedBuildPhaseTaskProducerBase, F
                 let path: Path
                 let dsymPath: Path?
                 let topLevelItemPath: Path?
+                let prefixes: [String]
                 if let settingsForRef, let presumedKind = kindFromSettings(settingsForRef), !useSearchPaths {
                     // If we have a Settings from a cross-project reference, use the _actual_ library path. This prevents downstream code from reconstituting the framework path by joining the framework path with the basename of the framework, which won't be correct for deep frameworks which also need the Versions/A path component.
-                    kind = presumedKind
+                    kind = presumedKind.kind
+                    prefixes = presumedKind.prefixes
                     path = settingsForRef.globalScope.evaluate(BuiltinMacros.TARGET_BUILD_DIR).join(settingsForRef.globalScope.evaluate(BuiltinMacros.EXECUTABLE_PATH)).normalize()
                     topLevelItemPath = absolutePath
                     if shouldGenerateDSYM(settingsForRef.globalScope) {
@@ -563,6 +568,7 @@ package final class SourcesTaskProducer: FilesBasedBuildPhaseTaskProducerBase, F
                     path = absolutePath
                     topLevelItemPath = nil
                     dsymPath = nil
+                    prefixes = []
                 }
 
                 return LinkerSpec.LibrarySpecifier(
@@ -572,6 +578,7 @@ package final class SourcesTaskProducer: FilesBasedBuildPhaseTaskProducerBase, F
                     useSearchPaths: useSearchPaths,
                     swiftModulePaths: [:],
                     swiftModuleAdditionalLinkerArgResponseFilePaths: [:],
+                    prefixes: prefixes,
                     topLevelItemPath: topLevelItemPath,
                     dsymPath: dsymPath,
                     privacyFile: privacyFile
@@ -581,7 +588,7 @@ package final class SourcesTaskProducer: FilesBasedBuildPhaseTaskProducerBase, F
                     kind: .object,
                     path: absolutePath,
                     mode: buildFile.shouldLinkWeakly ? .weak : .normal,
-                    useSearchPaths: useSearchPaths,
+                    useSearchPaths: false,
                     swiftModulePaths: swiftModulePaths,
                     swiftModuleAdditionalLinkerArgResponseFilePaths: swiftModuleAdditionalLinkerArgResponseFilePaths,
                     privacyFile: privacyFile
@@ -621,10 +628,16 @@ package final class SourcesTaskProducer: FilesBasedBuildPhaseTaskProducerBase, F
                 }
 
                 let libraryKind: LinkerSpec.LibrarySpecifier.Kind
+                let prefixes: [String]
                 switch library.libraryType {
-                case .framework: libraryKind = .framework; break
-                case .dynamicLibrary: libraryKind = .dynamic; break
-                case .staticLibrary: libraryKind = .static; break
+                case .framework: libraryKind = .framework; prefixes = []
+                case .dynamicLibrary:
+                    libraryKind = .dynamic;
+                    prefixes = context.lookupFileType(identifier: "compiled.mach-o.dylib")?.prefixes ?? []
+                case .staticLibrary:
+                    libraryKind = .static
+                    prefixes = context.lookupFileType(identifier: "archive.ar")?.prefixes ?? []
+                break
                 case let .unknown(fileExtension):
                     // An error of type this type should have already been manifested.
                     assertionFailure("unknown xcframework type: \(fileExtension)")
@@ -651,6 +664,7 @@ package final class SourcesTaskProducer: FilesBasedBuildPhaseTaskProducerBase, F
                     useSearchPaths: useSearchPaths,
                     swiftModulePaths: [:],
                     swiftModuleAdditionalLinkerArgResponseFilePaths: [:],
+                    prefixes: prefixes,
                     explicitDependencies: outputFilePaths,
                     xcframeworkSourcePath: xcframeworkPath,
                     privacyFile: nil

Sources/SWBTestSupport/RunDestinationTestSupport.swift

@@ -320,7 +320,7 @@ extension RunDestinationInfo {
     /// An `Environment` object with `PATH` or `LD_LIBRARY_PATH` set appropriately pointing into the toolchain to be able to run a built Swift binary in tests.
     ///
     /// - note: On macOS, the OS provided Swift runtime is used, so `DYLD_LIBRARY_PATH` is never set for Mach-O destinations.
-    package func hostRuntimeEnvironment(_ core: Core, initialEnvironment: Environment = Environment()) -> Environment {
+    package func hostRuntimeEnvironment(_ core: Core, initialEnvironment: Environment = Environment()) throws -> Environment {
         var environment = initialEnvironment
         guard let toolchain = core.toolchainRegistry.defaultToolchain else {
             return environment
@@ -329,7 +329,10 @@ extension RunDestinationInfo {
         case .elf:
             environment.prependPath(key: "LD_LIBRARY_PATH", value: toolchain.path.join("usr/lib/swift/\(platform)").str)
         case .pe:
-            environment.prependPath(key: .path, value: core.developerPath.path.join("Runtimes").join(toolchain.version.description).join("usr/bin").str)
+            let currentPath = Environment.current[.path] ?? ""
+            let runtimePath = core.developerPath.path.join("Runtimes").join(toolchain.version.description).join("usr/bin").str
+            let newPath = currentPath.isEmpty ? "\(runtimePath)" : "\(runtimePath);\(currentPath)"
+            environment[.path] = newPath
         case .macho:
             // Fall back to the OS provided Swift runtime
             break