[6.0] Improvements to prebuilt/provided library handling (#7713)

Cherry-pick of
#7496 and
#7433, excluding
[16b4142](16b4142).

**Explanation**: The idea is to enable provided libraries to be a part
of a toolchain, which means that instead of completely eliding them from
a package graph, they need to be handled post-resolution in a special
way and reflected in a build plan for Swift targets (as compiler and
linker arguments). Unified handling of `config.json` and
`provided-libraries.json` resources and dropped use of `Bundle` type
(config should always be a part of SDK or a toolchain).
**Scope**: Localized, touches modules graph resolution and build
planning only in certain configurations.
**Risk**: Low. While the change has a somewhat broad scope, it's NFC for
most builds and has been incubated on `main` for 2 months with no known
issues.
**Testing**: New test cases added and existing ones updated.
**Issue**: rdar://125531670
**Reviewer**: @MaxDesiatov and @bnbarham

---------

Co-authored-by: Pavel Yaskevich <[email protected]>

Author: MaxDesiatov

Sources/Build/BuildDescription/ProductBuildDescription.swift

@@ -46,6 +46,9 @@ public final class ProductBuildDescription: SPMBuildCore.ProductBuildDescription
     // Computed during build planning.
     var dylibs: [ProductBuildDescription] = []
 
+    /// The list of provided libraries that are going to be used by this product.
+    var providedLibraries: [String: AbsolutePath] = [:]
+
     /// Any additional flags to be added. These flags are expected to be computed during build planning.
     var additionalFlags: [String] = []
 
@@ -157,6 +160,8 @@ public final class ProductBuildDescription: SPMBuildCore.ProductBuildDescription
             args += ["-F", self.buildParameters.buildPath.pathString]
         }
 
+        self.providedLibraries.forEach { args += ["-L", $1.pathString, "-l", $0] }
+
         args += ["-L", self.buildParameters.buildPath.pathString]
         args += try ["-o", binaryPath.pathString]
         args += ["-module-name", self.product.name.spm_mangledToC99ExtendedIdentifier()]

Sources/Build/BuildManifest/LLBuildManifestBuilder+Swift.swift

@@ -429,6 +429,8 @@ extension LLBuildManifestBuilder {
             if target.underlying is BinaryTarget { return }
             // Ignore Plugin Targets.
             if target.underlying is PluginTarget { return }
+            // Ignore Provided Libraries.
+            if target.underlying is ProvidedLibraryTarget { return }
 
             // Depend on the binary for executable targets.
             if target.type == .executable && !prepareForIndexing {

Sources/Build/BuildOperation.swift

@@ -275,7 +275,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
 
     // TODO: Currently this function will only match frameworks.
     func detectUnexpressedDependencies(
-        availableLibraries: [LibraryMetadata],
+        availableLibraries: [ProvidedLibrary],
         targetDependencyMap: [String: [String]]?
     ) {
         // Ensure we only emit these once, regardless of how many builds are being done.
@@ -285,8 +285,8 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         Self.didEmitUnexpressedDependencies = true
 
         let availableFrameworks = Dictionary<String, PackageIdentity>(uniqueKeysWithValues: availableLibraries.compactMap {
-            if let identity = Set($0.identities.map(\.identity)).spm_only {
-                return ("\($0.productName!).framework", identity)
+            if let identity = Set($0.metadata.identities.map(\.identity)).spm_only {
+                return ("\($0.metadata.productName).framework", identity)
             } else {
                 return nil
             }

Sources/Build/BuildPlan/BuildPlan+Product.swift

@@ -116,6 +116,8 @@ extension BuildPlan {
         }
         buildProduct.libraryBinaryPaths = dependencies.libraryBinaryPaths
 
+        buildProduct.providedLibraries = dependencies.providedLibraries
+
         buildProduct.availableTools = dependencies.availableTools
     }
 
@@ -128,6 +130,7 @@ extension BuildPlan {
         staticTargets: [ResolvedModule],
         systemModules: [ResolvedModule],
         libraryBinaryPaths: Set<AbsolutePath>,
+        providedLibraries: [String: AbsolutePath],
         availableTools: [String: AbsolutePath]
     ) {
         /* Prior to tools-version 5.9, we used to erroneously recursively traverse executable/plugin dependencies and statically include their
@@ -205,6 +208,7 @@ extension BuildPlan {
         var staticTargets = [ResolvedModule]()
         var systemModules = [ResolvedModule]()
         var libraryBinaryPaths: Set<AbsolutePath> = []
+        var providedLibraries = [String: AbsolutePath]()
         var availableTools = [String: AbsolutePath]()
 
         for dependency in allTargets {
@@ -258,6 +262,8 @@ extension BuildPlan {
                     }
                 case .plugin:
                     continue
+                case .providedLibrary:
+                    providedLibraries[target.name] = target.underlying.path
                 }
 
             case .product(let product, _):
@@ -275,7 +281,7 @@ extension BuildPlan {
             }
         }
 
-        return (linkLibraries, staticTargets, systemModules, libraryBinaryPaths, availableTools)
+        return (linkLibraries, staticTargets, systemModules, libraryBinaryPaths, providedLibraries, availableTools)
     }
 
     /// Extracts the artifacts  from an artifactsArchive

Sources/Build/BuildPlan/BuildPlan+Swift.swift

@@ -14,6 +14,7 @@ import struct Basics.InternalError
 import class PackageModel.BinaryTarget
 import class PackageModel.ClangTarget
 import class PackageModel.SystemLibraryTarget
+import class PackageModel.ProvidedLibraryTarget
 
 extension BuildPlan {
     func plan(swiftTarget: SwiftTargetBuildDescription) throws {
@@ -48,6 +49,10 @@ extension BuildPlan {
                         swiftTarget.libraryBinaryPaths.insert(library.libraryPath)
                     }
                 }
+            case let target as ProvidedLibraryTarget:
+                swiftTarget.additionalFlags += [
+                    "-I", target.path.pathString
+                ]
             default:
                 break
             }

Sources/Build/BuildPlan/BuildPlan.swift

@@ -419,7 +419,7 @@ public class BuildPlan: SPMBuildCore.BuildPlan {
                     toolsVersion: toolsVersion,
                     fileSystem: fileSystem
                 ))
-            case is SystemLibraryTarget, is BinaryTarget:
+            case is SystemLibraryTarget, is BinaryTarget, is ProvidedLibraryTarget:
                 break
             default:
                 throw InternalError("unhandled \(target.underlying)")

Sources/Commands/PackageCommands/EditCommands.swift

@@ -72,7 +72,6 @@ extension SwiftPackageCommand {
                 packageName: packageName,
                 forceRemove: shouldForceRemove,
                 root: swiftCommandState.getWorkspaceRoot(),
-                availableLibraries: swiftCommandState.getHostToolchain().providedLibraries,
                 observabilityScope: swiftCommandState.observabilityScope
             )
         }

Sources/Commands/PackageCommands/Update.swift

@@ -71,7 +71,7 @@ extension SwiftPackageCommand {
                 case .removed:
                     report += "\n"
                     report += "- \(package.identity) \(currentVersion)"
-                case .unchanged:
+                case .unchanged, .usesLibrary:
                     continue
                 }
             }

Sources/Commands/Snippets/Cards/TopCard.swift

@@ -151,6 +151,8 @@ fileprivate extension Target.Kind {
             return "snippets"
         case .macro:
             return "macros"
+        case .providedLibrary:
+            return "provided libraries"
         }
     }
 }

Sources/CoreCommands/SwiftCommandState.swift

@@ -622,7 +622,6 @@ public final class SwiftCommandState {
                 explicitProduct: explicitProduct,
                 forceResolvedVersions: options.resolver.forceResolvedVersions,
                 testEntryPointPath: testEntryPointPath,
-                availableLibraries: self.getHostToolchain().providedLibraries,
                 observabilityScope: self.observabilityScope
             )