Skip to content

Wire up support for module wrapping on non-Darwin platforms #145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 18, 2025
Merged

Wire up support for module wrapping on non-Darwin platforms #145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions Sources/SWBAndroidPlatform/Plugin.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,6 @@ struct AndroidSDKRegistryExtension: SDKRegistryExtension {
let defaultProperties: [String: PropertyListItem] = [
"SDK_STAT_CACHE_ENABLE": "NO",

// Workaround to avoid `-add_ast_path` on Linux, apparently this needs to perform some "swift modulewrap" step instead.
"GCC_GENERATE_DEBUGGING_SYMBOLS": .plString("NO"),

// Workaround to avoid `-dependency_info` on Linux.
"LD_DEPENDENCY_INFO_FILE": .plString(""),

Expand Down
3 changes: 0 additions & 3 deletions Sources/SWBCore/SDKRegistry.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -749,9 +749,6 @@ public final class SDKRegistry: SDKRegistryLookup, CustomStringConvertible, Send
switch operatingSystem {
case .linux:
defaultProperties = [
// Workaround to avoid `-add_ast_path` on Linux, apparently this needs to perform some "swift modulewrap" step instead.
"GCC_GENERATE_DEBUGGING_SYMBOLS": .plString("NO"),

// Workaround to avoid `-dependency_info` on Linux.
"LD_DEPENDENCY_INFO_FILE": .plString(""),

Expand Down
2 changes: 2 additions & 0 deletions Sources/SWBCore/Settings/BuiltinMacros.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -976,6 +976,7 @@ public final class BuiltinMacros {
public static let SUPPORTS_TEXT_BASED_API = BuiltinMacros.declareBooleanMacro("SUPPORTS_TEXT_BASED_API")
public static let SWIFT_AUTOLINK_EXTRACT_OUTPUT_PATH = BuiltinMacros.declarePathMacro("SWIFT_AUTOLINK_EXTRACT_OUTPUT_PATH")
public static let PLATFORM_REQUIRES_SWIFT_AUTOLINK_EXTRACT = BuiltinMacros.declareBooleanMacro("PLATFORM_REQUIRES_SWIFT_AUTOLINK_EXTRACT")
public static let PLATFORM_REQUIRES_SWIFT_MODULEWRAP = BuiltinMacros.declareBooleanMacro("PLATFORM_REQUIRES_SWIFT_MODULEWRAP")
public static let SWIFT_ABI_CHECKER_BASELINE_DIR = BuiltinMacros.declareStringMacro("SWIFT_ABI_CHECKER_BASELINE_DIR")
public static let SWIFT_ABI_CHECKER_EXCEPTIONS_FILE = BuiltinMacros.declareStringMacro("SWIFT_ABI_CHECKER_EXCEPTIONS_FILE")
public static let SWIFT_ABI_GENERATION_TOOL_OUTPUT_DIR = BuiltinMacros.declareStringMacro("SWIFT_ABI_GENERATION_TOOL_OUTPUT_DIR")
Expand Down Expand Up @@ -2128,6 +2129,7 @@ public final class BuiltinMacros {
SUPPORTS_TEXT_BASED_API,
SWIFT_AUTOLINK_EXTRACT_OUTPUT_PATH,
PLATFORM_REQUIRES_SWIFT_AUTOLINK_EXTRACT,
PLATFORM_REQUIRES_SWIFT_MODULEWRAP,
SWIFT_ABI_CHECKER_BASELINE_DIR,
SWIFT_ABI_CHECKER_EXCEPTIONS_FILE,
SWIFT_ABI_GENERATION_TOOL_OUTPUT_DIR,
Expand Down
1 change: 1 addition & 0 deletions Sources/SWBCore/Settings/Settings.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2535,6 +2535,7 @@ private class SettingsBuilder {

sdkTable.push(BuiltinMacros.DYNAMIC_LIBRARY_EXTENSION, literal: imageFormat.dynamicLibraryExtension)
sdkTable.push(BuiltinMacros.PLATFORM_REQUIRES_SWIFT_AUTOLINK_EXTRACT, literal: imageFormat.requiresSwiftAutolinkExtract)
sdkTable.push(BuiltinMacros.PLATFORM_REQUIRES_SWIFT_MODULEWRAP, literal: imageFormat.requiresSwiftModulewrap)
}

// Add additional SDK default settings.
Expand Down
2 changes: 1 addition & 1 deletion Sources/SWBCore/Specs/Tools/LinkerTools.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -592,7 +592,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType, @unchec
// If we are linking Swift and build for debugging, pass the right .swiftmodule file for the current architecture to the
// linker. This is needed so that debugging these modules works correctly. Note that `swiftModulePaths` will be empty for
// anything but static archives and object files, because dynamic libraries and frameworks do not require this.
if isLinkUsingSwift && cbc.scope.evaluate(BuiltinMacros.GCC_GENERATE_DEBUGGING_SYMBOLS) {
if isLinkUsingSwift && cbc.scope.evaluate(BuiltinMacros.GCC_GENERATE_DEBUGGING_SYMBOLS) && !cbc.scope.evaluate(BuiltinMacros.PLATFORM_REQUIRES_SWIFT_MODULEWRAP) {
for library in libraries {
if let swiftModulePath = library.swiftModulePaths[cbc.scope.evaluate(BuiltinMacros.CURRENT_ARCH)] {
commandLine += ["-Xlinker", "-add_ast_path", "-Xlinker", swiftModulePath.str]
Expand Down
7 changes: 6 additions & 1 deletion Sources/SWBCore/Specs/Tools/SwiftCompiler.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2127,6 +2127,11 @@ public final class SwiftCompilerSpec : CompilerSpec, SpecIdentifierType, SwiftDi
return (inputs, outputs)
}()

if cbc.scope.evaluate(BuiltinMacros.PLATFORM_REQUIRES_SWIFT_MODULEWRAP) && cbc.scope.evaluate(BuiltinMacros.GCC_GENERATE_DEBUGGING_SYMBOLS) {
let moduleWrapOutput = Path(moduleFilePath.withoutSuffix + ".o")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Windows really should use .obj instead of .o.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I filed #149 to track this because it needs to be done in a few other places at the same time to avoid a regression

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@compnerd I think it's more like "MSVC ABI should use .obj and gcc/mingw ABI should use .o", right?

moduleOutputPaths.append(moduleWrapOutput)
}

// Add const metadata outputs to extra compilation outputs
if await supportConstSupplementaryMetadata(cbc, delegate, compilationMode: compilationMode) {
// If using whole module optimization then we use the -master.swiftconstvalues file from the sole compilation task.
Expand Down Expand Up @@ -2963,7 +2968,7 @@ public final class SwiftCompilerSpec : CompilerSpec, SpecIdentifierType, SwiftDi
// be a source-less target which just contains object files in it's framework phase.
let currentPlatformFilter = PlatformFilter(scope)
let containsSources = (producer.configuredTarget?.target as? StandardTarget)?.sourcesBuildPhase?.buildFiles.filter { currentPlatformFilter.matches($0.platformFilters) }.isEmpty == false
if containsSources && inputFileTypes.contains(where: { $0.conformsTo(identifier: "sourcecode.swift") }) && scope.evaluate(BuiltinMacros.GCC_GENERATE_DEBUGGING_SYMBOLS) {
if containsSources && inputFileTypes.contains(where: { $0.conformsTo(identifier: "sourcecode.swift") }) && scope.evaluate(BuiltinMacros.GCC_GENERATE_DEBUGGING_SYMBOLS) && !scope.evaluate(BuiltinMacros.PLATFORM_REQUIRES_SWIFT_MODULEWRAP) {
let moduleName = scope.evaluate(BuiltinMacros.SWIFT_MODULE_NAME)
let moduleFileDir = scope.evaluate(BuiltinMacros.PER_ARCH_MODULE_FILE_DIR)
let moduleFilePath = moduleFileDir.join(moduleName + ".swiftmodule")
Expand Down
9 changes: 9 additions & 0 deletions Sources/SWBUtil/ProcessInfo.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,15 @@ extension ImageFormat {
return false
}
}

public var requiresSwiftModulewrap: Bool {
switch self {
case .macho:
return false
default:
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: can we prefer to list the cases explicitly? I like to stay away from default since it forces re-evaluation of use sites when new cases are added. Not that adding new formats is particularly likely, but I could at least see a new one for wasm being added in the future.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is really an is-not-macho check, I don't expect we will ever move away from module wrap for any future object formats

return true
}
}
}

extension FixedWidthInteger {
Expand Down
3 changes: 0 additions & 3 deletions Sources/SWBWebAssemblyPlatform/Plugin.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,6 @@ struct WebAssemblySDKRegistryExtension: SDKRegistryExtension {
let defaultProperties: [String: PropertyListItem] = [
"SDK_STAT_CACHE_ENABLE": "NO",

// Workaround to avoid `-add_ast_path` on WebAssembly, apparently this needs to perform some "swift modulewrap" step instead.
"GCC_GENERATE_DEBUGGING_SYMBOLS": .plString("NO"),

"GENERATE_TEXT_BASED_STUBS": "NO",
"GENERATE_INTERMEDIATE_TEXT_BASED_STUBS": "NO",

Expand Down
174 changes: 174 additions & 0 deletions Tests/SWBBuildSystemTests/BuildOperationTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,180 @@ fileprivate struct BuildOperationTests: CoreBasedTests {
}
}

@Test(.requireSDKs(.host), .requireThreadSafeWorkingDirectory)
func debuggableCommandLineTool() async throws {
try await withTemporaryDirectory { (tmpDir: Path) in
let testProject = try await TestProject(
"TestProject",
sourceRoot: tmpDir,
groupTree: TestGroup(
"SomeFiles",
children: [
TestFile("main.swift"),
TestFile("dynamic.swift"),
TestFile("static.swift"),
]),
buildConfigurations: [
TestBuildConfiguration("Debug", buildSettings: [
"ARCHS": "$(ARCHS_STANDARD)",
"CODE_SIGNING_ALLOWED": ProcessInfo.processInfo.hostOperatingSystem() == .macOS ? "YES" : "NO",
"CODE_SIGN_IDENTITY": "-",
"CODE_SIGN_ENTITLEMENTS": "Entitlements.plist",
"DEFINES_MODULE": "YES",
"PRODUCT_NAME": "$(TARGET_NAME)",
"SDKROOT": "$(HOST_PLATFORM)",
"SUPPORTED_PLATFORMS": "$(HOST_PLATFORM)",
"SWIFT_VERSION": swiftVersion,
"GCC_GENERATE_DEBUGGING_SYMBOLS": "YES",
])
],
targets: [
TestStandardTarget(
"tool",
type: .commandLineTool,
buildConfigurations: [
TestBuildConfiguration("Debug", buildSettings: [
"LD_RUNPATH_SEARCH_PATHS": "@loader_path/",
])
],
buildPhases: [
TestSourcesBuildPhase(["main.swift"]),
TestFrameworksBuildPhase([
TestBuildFile(.target("dynamiclib")),
TestBuildFile(.target("staticlib")),
])
],
dependencies: [
"dynamiclib",
"staticlib",
]
),
TestStandardTarget(
"dynamiclib",
type: .dynamicLibrary,
buildConfigurations: [
TestBuildConfiguration("Debug", buildSettings: [
"DYLIB_INSTALL_NAME_BASE": "$ORIGIN",
"DYLIB_INSTALL_NAME_BASE[sdk=macosx*]": "@rpath",

// FIXME: Find a way to make these default
"EXECUTABLE_PREFIX": "lib",
"EXECUTABLE_PREFIX[sdk=windows*]": "",
])
],
buildPhases: [
TestSourcesBuildPhase(["dynamic.swift"]),
]
),
TestStandardTarget(
"staticlib",
type: .staticLibrary,
buildConfigurations: [
TestBuildConfiguration("Debug", buildSettings: [
// FIXME: Find a way to make these default
"EXECUTABLE_PREFIX": "lib",
"EXECUTABLE_PREFIX[sdk=windows*]": "",
])
],
buildPhases: [
TestSourcesBuildPhase(["static.swift"]),
]
),
])
let core = try await getCore()
let tester = try await BuildOperationTester(core, testProject, simulated: false)

let projectDir = tester.workspace.projects[0].sourceRoot

try await tester.fs.writeFileContents(projectDir.join("main.swift")) { stream in
stream <<< "import dynamiclib\n"
stream <<< "import staticlib\n"
stream <<< "dynamicLib()\n"
stream <<< "dynamicLib()\n"
stream <<< "staticLib()\n"
stream <<< "print(\"Hello world\")\n"
}

try await tester.fs.writeFileContents(projectDir.join("dynamic.swift")) { stream in
stream <<< "public func dynamicLib() { }"
}

try await tester.fs.writeFileContents(projectDir.join("static.swift")) { stream in
stream <<< "public func staticLib() { }"
}

try await tester.fs.writePlist(projectDir.join("Entitlements.plist"), .plDict([:]))

let provisioningInputs = [
"dynamiclib": ProvisioningTaskInputs(identityHash: "-", signedEntitlements: .plDict([:]), simulatedEntitlements: .plDict([:])),
"staticlib": ProvisioningTaskInputs(identityHash: "-", signedEntitlements: .plDict([:]), simulatedEntitlements: .plDict([:])),
"tool": ProvisioningTaskInputs(identityHash: "-", signedEntitlements: .plDict([:]), simulatedEntitlements: .plDict([:]))
]

let destination: RunDestinationInfo = .host
try await tester.checkBuild(runDestination: destination, persistent: true, signableTargets: Set(provisioningInputs.keys), signableTargetInputs: provisioningInputs) { results in
results.checkNoErrors()
if core.hostOperatingSystem.imageFormat.requiresSwiftModulewrap {
try results.checkTask(.matchTargetName("tool"), .matchRulePattern(["WriteAuxiliaryFile", .suffix("LinkFileList")])) { task in
let auxFileAction = try #require(task.action as? AuxiliaryFileTaskAction)
let contents = try tester.fs.read(auxFileAction.context.input).asString
let files = contents.components(separatedBy: "\n").map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }.filter { !$0.isEmpty }
#expect(files.count == 2)
#expect(files[0].hasSuffix("tool.o"))
#expect(files[1].hasSuffix("main.o"))
}
let toolWrap = try #require(results.getTask(.matchTargetName("tool"), .matchRuleType("SwiftModuleWrap")))
try results.checkTask(.matchTargetName("tool"), .matchRuleType("Ld")) { task in
try results.checkTaskFollows(task, toolWrap)
}

try results.checkTask(.matchTargetName("dynamiclib"), .matchRulePattern(["WriteAuxiliaryFile", .suffix("LinkFileList")])) { task in
let auxFileAction = try #require(task.action as? AuxiliaryFileTaskAction)
let contents = try tester.fs.read(auxFileAction.context.input).asString
let files = contents.components(separatedBy: "\n").map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }.filter { !$0.isEmpty }
#expect(files.count == 2)
#expect(files[0].hasSuffix("dynamiclib.o"))
#expect(files[1].hasSuffix("dynamic.o"))
}
let dylibWrap = try #require(results.getTask(.matchTargetName("dynamiclib"), .matchRuleType("SwiftModuleWrap")))
try results.checkTask(.matchTargetName("dynamiclib"), .matchRuleType("Ld")) { task in
try results.checkTaskFollows(task, dylibWrap)
}

try results.checkTask(.matchTargetName("staticlib"), .matchRulePattern(["WriteAuxiliaryFile", .suffix("LinkFileList")])) { task in
let auxFileAction = try #require(task.action as? AuxiliaryFileTaskAction)
let contents = try tester.fs.read(auxFileAction.context.input).asString
let files = contents.components(separatedBy: "\n").map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }.filter { !$0.isEmpty }
#expect(files.count == 2)
#expect(files[0].hasSuffix("staticlib.o"))
#expect(files[1].hasSuffix("static.o"))
}
let staticWrap = try #require(results.getTask(.matchTargetName("staticlib"), .matchRuleType("SwiftModuleWrap")))
try results.checkTask(.matchTargetName("staticlib"), .matchRuleType("Libtool")) { task in
try results.checkTaskFollows(task, staticWrap)
}
}

let toolchain = try #require(try await getCore().toolchainRegistry.defaultToolchain)
let environment: [String: String]
if destination.platform == "linux" {
environment = ["LD_LIBRARY_PATH": toolchain.path.join("usr/lib/swift/linux").str]
} else {
environment = [:]
}

let executionResult = try await Process.getOutput(url: URL(fileURLWithPath: projectDir.join("build").join("Debug\(destination.builtProductsDirSuffix)").join(core.hostOperatingSystem.imageFormat.executableName(basename: "tool")).str), arguments: [], environment: environment)
#expect(executionResult.exitStatus == .exit(0))
if core.hostOperatingSystem == .windows {
#expect(String(decoding: executionResult.stdout, as: UTF8.self) == "Hello world\r\n")
} else {
#expect(String(decoding: executionResult.stdout, as: UTF8.self) == "Hello world\n")
}
#expect(String(decoding: executionResult.stderr, as: UTF8.self) == "")
}
}
}

/// Check that environment variables are propagated from the user environment correctly.
@Test(.requireSDKs(.host), .skipHostOS(.windows), .requireSystemPackages(apt: "yacc", yum: "byacc"))
func userEnvironment() async throws {
Expand Down