Skip to content

Commit aedc2fa

Browse files
committed
[Dependency Scanning] Serialized Swift binary module serialized search paths
1 parent 90f2fba commit aedc2fa

File tree

5 files changed

+183
-9
lines changed

5 files changed

+183
-9
lines changed

include/swift/AST/DiagnosticsSema.def

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2379,7 +2379,7 @@ NOTE(dependency_as_imported_by_main_module,none,
23792379
NOTE(dependency_as_imported_by, none,
23802380
"a dependency of %select{Swift|Clang}2 module '%0': '%1'", (StringRef, StringRef, bool))
23812381
NOTE(inherited_search_path_resolves_module,none,
2382-
"'%0' can be found on search path used to build module '%1': '%2'. "
2382+
"'%0' can be found on a search path used to build module '%1': '%2'. "
23832383
"These search paths are not inherited by the current compilation.", (StringRef, StringRef, StringRef))
23842384
ERROR(clang_dependency_scan_error, none, "Clang dependency scanner failure: %0", (StringRef))
23852385
ERROR(clang_header_dependency_scan_error, none, "Bridging header dependency scan failure: %0", (StringRef))

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ using llvm::BCVBR;
4141
const unsigned char MODULE_DEPENDENCY_CACHE_FORMAT_SIGNATURE[] = {'I', 'M', 'D','C'};
4242
const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR = 9;
4343
/// Increment this on every change.
44-
const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 2;
44+
const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 3;
4545

4646
/// Various identifiers in this format will rely on having their strings mapped
4747
/// using this ID.
@@ -78,6 +78,7 @@ using ModuleCacheKeyIDField = IdentifierIDField;
7878
using ImportArrayIDField = IdentifierIDField;
7979
using LinkLibrariesArrayIDField = IdentifierIDField;
8080
using MacroDependenciesArrayIDField = IdentifierIDField;
81+
using SearchPathArrayIDField = IdentifierIDField;
8182
using FlagIDArrayIDField = IdentifierIDField;
8283
using DependencyIDArrayIDField = IdentifierIDField;
8384
using SourceLocationIDArrayIDField = IdentifierIDField;
@@ -101,6 +102,8 @@ enum {
101102
LINK_LIBRARY_ARRAY_NODE,
102103
MACRO_DEPENDENCY_NODE,
103104
MACRO_DEPENDENCY_ARRAY_NODE,
105+
SEARCH_PATH_NODE,
106+
SEARCH_PATH_ARRAY_NODE,
104107
IMPORT_STATEMENT_NODE,
105108
IMPORT_STATEMENT_ARRAY_NODE,
106109
OPTIONAL_IMPORT_STATEMENT_ARRAY_NODE,
@@ -169,6 +172,17 @@ using MacroDependencyLayout =
169172
using MacroDependencyArrayLayout =
170173
BCRecordLayout<MACRO_DEPENDENCY_ARRAY_NODE, IdentifierIDArryField>;
171174

175+
// A record for a serialized search pathof a given dependency
176+
// node (Swift binary module dependency only).
177+
using SearchPathLayout =
178+
BCRecordLayout<SEARCH_PATH_NODE, // ID
179+
IdentifierIDField, // path
180+
IsFrameworkField, // isFramework
181+
IsSystemField // isSystem
182+
>;
183+
using SearchPathArrayLayout =
184+
BCRecordLayout<SEARCH_PATH_ARRAY_NODE, IdentifierIDArryField>;
185+
172186
// A record capturing information about a given 'import' statement
173187
// captured in a dependency node, including its source location.
174188
using ImportStatementLayout =
@@ -248,6 +262,7 @@ using SwiftBinaryModuleDetailsLayout =
248262
FileIDField, // definingInterfacePath
249263
IdentifierIDField, // headerModuleDependencies
250264
FileIDArrayIDField, // headerSourceFiles
265+
SearchPathArrayIDField, // serializedSearchPaths
251266
IsFrameworkField, // isFramework
252267
IsStaticField, // isStatic
253268
IdentifierIDField, // moduleCacheKey

lib/DependencyScan/ModuleDependencyCacheSerialization.cpp

Lines changed: 121 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,11 @@ class ModuleDependenciesCacheDeserializer {
4545
std::vector<LinkLibrary> LinkLibraries;
4646
std::vector<std::vector<uint64_t>> ArraysOfLinkLibraryIDs;
4747
std::vector<std::pair<std::string, MacroPluginDependency>> MacroDependencies;
48+
std::vector<serialization::SearchPath> SearchPaths;
4849
std::vector<std::vector<uint64_t>> ArraysOfMacroDependenciesIDs;
4950
std::vector<ScannerImportStatementInfo> ImportStatements;
5051
std::vector<std::vector<uint64_t>> ArraysOfImportStatementIDs;
52+
std::vector<std::vector<uint64_t>> ArraysOfSearchPathIDs;
5153
std::vector<std::vector<uint64_t>> ArraysOfOptionalImportStatementIDs;
5254

5355
llvm::BitstreamCursor Cursor;
@@ -66,6 +68,8 @@ class ModuleDependenciesCacheDeserializer {
6668
std::optional<std::vector<LinkLibrary>> getLinkLibraryArray(unsigned n);
6769
std::optional<std::vector<std::pair<std::string, MacroPluginDependency>>>
6870
getMacroDependenciesArray(unsigned n);
71+
std::optional<std::vector<serialization::SearchPath>>
72+
getSearchPathArray(unsigned n);
6973
std::optional<std::vector<ScannerImportStatementInfo>>
7074
getImportStatementInfoArray(unsigned n);
7175
std::optional<std::vector<ScannerImportStatementInfo>>
@@ -389,6 +393,24 @@ bool ModuleDependenciesCacheDeserializer::readGraph(
389393
break;
390394
}
391395

396+
case SEARCH_PATH_NODE: {
397+
unsigned pathStrID;
398+
bool isFramework, isSystem;
399+
SearchPathLayout::readRecord(Scratch, pathStrID, isFramework, isSystem);
400+
auto pathStr = getIdentifier(pathStrID);
401+
if (!pathStr)
402+
llvm::report_fatal_error("Bad search path: no path string");
403+
SearchPaths.push_back({*pathStr, isFramework, isSystem});
404+
break;
405+
}
406+
407+
case SEARCH_PATH_ARRAY_NODE: {
408+
ArrayRef<uint64_t> identifierIDs;
409+
SearchPathArrayLayout::readRecord(Scratch, identifierIDs);
410+
ArraysOfSearchPathIDs.push_back(identifierIDs.vec());
411+
break;
412+
}
413+
392414
case IMPORT_STATEMENT_NODE: {
393415
unsigned importIdentifierID, bufferIdentifierID;
394416
unsigned lineNumber, columnNumber;
@@ -653,14 +675,15 @@ bool ModuleDependenciesCacheDeserializer::readGraph(
653675
llvm::report_fatal_error(
654676
"Unexpected SWIFT_BINARY_MODULE_DETAILS_NODE record");
655677
unsigned compiledModulePathID, moduleDocPathID, moduleSourceInfoPathID,
656-
headerImportID, definingInterfacePathID,
678+
headerImportID, definingInterfacePathID, searchPathArrayID,
657679
headerModuleDependenciesArrayID, headerImportsSourceFilesArrayID,
658680
isFramework, isStatic, moduleCacheKeyID, userModuleVersionID;
659681
SwiftBinaryModuleDetailsLayout::readRecord(
660682
Scratch, compiledModulePathID, moduleDocPathID,
661683
moduleSourceInfoPathID, headerImportID, definingInterfacePathID,
662684
headerModuleDependenciesArrayID, headerImportsSourceFilesArrayID,
663-
isFramework, isStatic, moduleCacheKeyID, userModuleVersionID);
685+
searchPathArrayID, isFramework, isStatic, moduleCacheKeyID,
686+
userModuleVersionID);
664687

665688
auto compiledModulePath = getIdentifier(compiledModulePathID);
666689
if (!compiledModulePath)
@@ -685,14 +708,17 @@ bool ModuleDependenciesCacheDeserializer::readGraph(
685708
if (!definingInterfacePath)
686709
llvm::report_fatal_error(
687710
"Bad binary direct dependencies: no defining interface path");
711+
auto searchPaths = getSearchPathArray(searchPathArrayID);
712+
if (!searchPaths)
713+
llvm::report_fatal_error(
714+
"Bad binary direct dependencies: no serialized search paths");
688715

689716
// Form the dependencies storage object
690717
auto moduleDep = ModuleDependencyInfo::forSwiftBinaryModule(
691718
*compiledModulePath, *moduleDocPath, *moduleSourceInfoPath,
692719
importStatements, optionalImportStatements, linkLibraries,
693-
{}, // TODO: serialized search path serialization
694-
*headerImport, *definingInterfacePath, isFramework, isStatic,
695-
*moduleCacheKey, *userModuleVersion);
720+
*searchPaths, *headerImport, *definingInterfacePath, isFramework,
721+
isStatic, *moduleCacheKey, *userModuleVersion);
696722

697723
addCommonDependencyInfo(moduleDep);
698724
addSwiftCommonDependencyInfo(moduleDep);
@@ -909,6 +935,25 @@ ModuleDependenciesCacheDeserializer::getMacroDependenciesArray(unsigned n) {
909935
return result;
910936
}
911937

938+
std::optional<std::vector<serialization::SearchPath>>
939+
ModuleDependenciesCacheDeserializer::getSearchPathArray(unsigned n) {
940+
if (n == 0)
941+
return std::vector<serialization::SearchPath>();
942+
943+
--n;
944+
if (n >= ArraysOfSearchPathIDs.size())
945+
return std::nullopt;
946+
947+
auto &llIDs = ArraysOfSearchPathIDs[n];
948+
949+
auto IDtoLLMap = [this](unsigned index) { return SearchPaths[index]; };
950+
std::vector<serialization::SearchPath> result;
951+
result.reserve(llIDs.size());
952+
std::transform(llIDs.begin(), llIDs.end(), std::back_inserter(result),
953+
IDtoLLMap);
954+
return result;
955+
}
956+
912957
std::optional<std::vector<ScannerImportStatementInfo>>
913958
ModuleDependenciesCacheDeserializer::getImportStatementInfoArray(unsigned n) {
914959
if (n == 0)
@@ -1084,6 +1129,7 @@ class ModuleDependenciesCacheSerializer {
10841129

10851130
std::unordered_map<ModuleDependencyID, unsigned> LinkLibraryArrayIDsMap;
10861131
std::unordered_map<ModuleDependencyID, unsigned> MacroDependenciesArrayIDsMap;
1132+
std::unordered_map<ModuleDependencyID, unsigned> SearchPathArrayIDsMap;
10871133
std::unordered_map<ModuleDependencyID, unsigned> ImportInfosArrayIDsMap;
10881134
std::unordered_map<ModuleDependencyID, unsigned>
10891135
OptionalImportInfosArrayIDsMap;
@@ -1110,6 +1156,7 @@ class ModuleDependenciesCacheSerializer {
11101156
ModuleIdentifierArrayKind arrayKind) const;
11111157
unsigned getLinkLibrariesArrayID(ModuleDependencyID moduleID) const;
11121158
unsigned getMacroDependenciesArrayID(ModuleDependencyID moduleID) const;
1159+
unsigned getSearchPathArrayID(ModuleDependencyID moduleID) const;
11131160
unsigned getImportStatementsArrayID(ModuleDependencyID moduleID) const;
11141161
unsigned
11151162
getOptionalImportStatementsArrayID(ModuleDependencyID moduleID) const;
@@ -1146,6 +1193,10 @@ class ModuleDependenciesCacheSerializer {
11461193
unsigned writeMacroDependencies(const ModuleDependencyInfo &dependencyInfo);
11471194
void writeMacroDependenciesArray(unsigned startIndex, unsigned count);
11481195

1196+
void writeSearchPaths(const ModuleDependenciesCache &cache);
1197+
unsigned writeSearchPaths(const SwiftBinaryModuleDependencyStorage &dependencyInfo);
1198+
void writeSearchPathsArray(unsigned startIndex, unsigned count);
1199+
11491200
void writeImportStatementInfos(const ModuleDependenciesCache &cache);
11501201
unsigned writeImportStatementInfos(const ModuleDependencyInfo &dependencyInfo,
11511202
bool optional);
@@ -1206,6 +1257,8 @@ void ModuleDependenciesCacheSerializer::writeBlockInfoBlock() {
12061257
BLOCK_RECORD(graph_block, LINK_LIBRARY_ARRAY_NODE);
12071258
BLOCK_RECORD(graph_block, MACRO_DEPENDENCY_NODE);
12081259
BLOCK_RECORD(graph_block, MACRO_DEPENDENCY_ARRAY_NODE);
1260+
BLOCK_RECORD(graph_block, SEARCH_PATH_NODE);
1261+
BLOCK_RECORD(graph_block, SEARCH_PATH_ARRAY_NODE);
12091262
BLOCK_RECORD(graph_block, IMPORT_STATEMENT_NODE);
12101263
BLOCK_RECORD(graph_block, IMPORT_STATEMENT_ARRAY_NODE);
12111264
BLOCK_RECORD(graph_block, OPTIONAL_IMPORT_STATEMENT_ARRAY_NODE);
@@ -1367,6 +1420,52 @@ void ModuleDependenciesCacheSerializer::writeMacroDependenciesArray(
13671420
Out, ScratchRecord, AbbrCodes[MacroDependencyArrayLayout::Code], vec);
13681421
}
13691422

1423+
void ModuleDependenciesCacheSerializer::writeSearchPaths(const ModuleDependenciesCache &cache) {
1424+
unsigned lastSPIndex = 0;
1425+
std::map<ModuleDependencyID, std::pair<unsigned, unsigned>>
1426+
moduleSearchPathArrayMap;
1427+
1428+
auto modMap = cache.getDependenciesMap(ModuleDependencyKind::SwiftBinary);
1429+
for (const auto &entry : modMap) {
1430+
ModuleDependencyID moduleID = {entry.getKey().str(), ModuleDependencyKind::SwiftBinary};
1431+
auto optionalDependencyInfo = cache.findDependency(moduleID);
1432+
assert(optionalDependencyInfo && "Expected dependency info.");
1433+
auto dependencyInfo = *optionalDependencyInfo;
1434+
unsigned numSPs = writeSearchPaths(*dependencyInfo->getAsSwiftBinaryModule());
1435+
moduleSearchPathArrayMap.insert({moduleID, std::make_pair(lastSPIndex, numSPs)});
1436+
lastSPIndex += numSPs;
1437+
}
1438+
1439+
unsigned lastSPArrayIndex = 1;
1440+
for (const auto &entry : modMap) {
1441+
ModuleDependencyID moduleID = {entry.getKey().str(), ModuleDependencyKind::SwiftBinary};
1442+
auto entries = moduleSearchPathArrayMap.at(moduleID);
1443+
if (entries.second == 0)
1444+
continue;
1445+
writeSearchPathsArray(entries.first, entries.second);
1446+
SearchPathArrayIDsMap.insert({moduleID, lastSPArrayIndex++});
1447+
}
1448+
}
1449+
unsigned ModuleDependenciesCacheSerializer::writeSearchPaths(const SwiftBinaryModuleDependencyStorage &dependencyInfo) {
1450+
using namespace graph_block;
1451+
for (const auto &searchPath : dependencyInfo.serializedSearchPaths) {
1452+
SearchPathLayout::emitRecord(
1453+
Out, ScratchRecord, AbbrCodes[SearchPathLayout::Code],
1454+
getIdentifier(searchPath.Path),
1455+
searchPath.IsFramework,
1456+
searchPath.IsSystem);
1457+
}
1458+
return dependencyInfo.serializedSearchPaths.size();
1459+
}
1460+
1461+
void ModuleDependenciesCacheSerializer::writeSearchPathsArray(unsigned startIndex, unsigned count) {
1462+
using namespace graph_block;
1463+
std::vector<unsigned> vec(count);
1464+
std::iota(vec.begin(), vec.end(), startIndex);
1465+
SearchPathArrayLayout::emitRecord(
1466+
Out, ScratchRecord, AbbrCodes[SearchPathArrayLayout::Code], vec);
1467+
}
1468+
13701469
void ModuleDependenciesCacheSerializer::writeImportStatementInfos(
13711470
const ModuleDependenciesCache &cache) {
13721471
unsigned lastImportInfoIndex = 0;
@@ -1559,10 +1658,10 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(
15591658
getIdentifierArrayID(
15601659
moduleID,
15611660
ModuleIdentifierArrayKind::HeaderInputDependencySourceFiles),
1661+
getSearchPathArrayID(moduleID),
15621662
swiftBinDeps->isFramework, swiftBinDeps->isStatic,
15631663
getIdentifier(swiftBinDeps->moduleCacheKey),
15641664
getIdentifier(swiftBinDeps->userModuleVersion));
1565-
15661665
break;
15671666
}
15681667
case swift::ModuleDependencyKind::SwiftPlaceholder: {
@@ -1699,6 +1798,15 @@ unsigned ModuleDependenciesCacheSerializer::getMacroDependenciesArrayID(
16991798
return iter->second;
17001799
}
17011800

1801+
unsigned ModuleDependenciesCacheSerializer::getSearchPathArrayID(
1802+
ModuleDependencyID moduleID) const {
1803+
auto iter = SearchPathArrayIDsMap.find(moduleID);
1804+
if (iter == SearchPathArrayIDsMap.end())
1805+
return 0;
1806+
1807+
return iter->second;
1808+
}
1809+
17021810
unsigned ModuleDependenciesCacheSerializer::getImportStatementsArrayID(
17031811
ModuleDependencyID moduleID) const {
17041812
auto iter = ImportInfosArrayIDsMap.find(moduleID);
@@ -1820,6 +1928,8 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
18201928
moduleID,
18211929
ModuleIdentifierArrayKind::HeaderInputDependencySourceFiles,
18221930
swiftBinDeps->headerSourceFiles);
1931+
llvm::for_each(swiftBinDeps->serializedSearchPaths,
1932+
[this](auto &sp) { addIdentifier(sp.Path); });
18231933
break;
18241934
}
18251935
case swift::ModuleDependencyKind::SwiftPlaceholder: {
@@ -1899,6 +2009,8 @@ void ModuleDependenciesCacheSerializer::writeInterModuleDependenciesCache(
18992009
registerRecordAbbr<LinkLibraryArrayLayout>();
19002010
registerRecordAbbr<MacroDependencyLayout>();
19012011
registerRecordAbbr<MacroDependencyArrayLayout>();
2012+
registerRecordAbbr<SearchPathLayout>();
2013+
registerRecordAbbr<SearchPathArrayLayout>();
19022014
registerRecordAbbr<ImportStatementLayout>();
19032015
registerRecordAbbr<ImportStatementArrayLayout>();
19042016
registerRecordAbbr<ModuleInfoLayout>();
@@ -1933,6 +2045,9 @@ void ModuleDependenciesCacheSerializer::writeInterModuleDependenciesCache(
19332045
// Write all the arrays of macro dependency infos for this graph
19342046
writeMacroDependencies(cache);
19352047

2048+
// Write all the arrays of binary-module-serialized search paths
2049+
writeSearchPaths(cache);
2050+
19362051
// Write the core graph
19372052
for (auto kind = ModuleDependencyKind::FirstKind;
19382053
kind != ModuleDependencyKind::LastKind; ++kind) {
Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
// RUN: %empty-directory(%t)
2+
// RUN: %empty-directory(%t/module-cache)
3+
// RUN: %empty-directory(%t/deps)
4+
// RUN: %empty-directory(%t/moreDeps)
5+
// RUN: split-file %s %t
6+
7+
// RUN: %target-swift-frontend -emit-module -emit-module-path %t/deps/B.swiftmodule -module-cache-path %t/module-cache %t/B.swift -module-name B -I %t/moreDeps
8+
9+
// Put the dependency module into a location discoverable by the first scan which will succeed and serialize scanner state
10+
// RUN: cp %t/moreDeps/C.swiftinterface %t/deps/C.swiftinterface
11+
12+
// RUN: %target-swift-frontend -scan-dependencies -o %t/deps.json %t/client.swift -I %t/deps -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -Rdependency-scan-cache -serialize-dependency-scan-cache -load-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache &> %t/initial_output.txt
13+
14+
// Remove the 'C' dependency module into a location not discoverable by the second scan in order to trigger a failure and use serialized scanner state
15+
// to emit the diagnostic
16+
// RUN: rm %t/deps/C.swiftinterface
17+
18+
// RUN: %target-swift-frontend -scan-dependencies -o %t/deps.json %t/client.swift -I %t/deps -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -Rdependency-scan-cache -load-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -validate-prior-dependency-scan-cache &> %t/output.txt
19+
// RUN: cat %t/output.txt | %FileCheck %s
20+
21+
// CHECK: remark: Incremental module scan: Re-using serialized module scanning dependency cache from: '{{.*}}cache.moddepcache'.
22+
// CHECK: remark: Incremental module scan: Dependency info for module 'C' invalidated due to a modified input since last scan: '{{.*}}deps{{/|\\}}C.swiftinterface'.
23+
// CHECK: remark: Incremental module scan: Dependency info for module 'deps' invalidated due to an out-of-date dependency.
24+
// CHECK: error: Unable to find module dependency: 'C'
25+
// CHECK: note: a dependency of main module 'deps'
26+
// CHECK: note: 'C' can be found on a search path used to build module 'B': '{{.*}}moreDeps'. These search paths are not inherited by the current compilation.
27+
28+
//--- moreDeps/C.swiftinterface
29+
// swift-interface-format-version: 1.0
30+
// swift-module-flags: -module-name C -enable-library-evolution
31+
public struct structC {}
32+
33+
//--- deps/A.swiftinterface
34+
// swift-interface-format-version: 1.0
35+
// swift-module-flags: -module-name A -enable-library-evolution
36+
public func funcA() {}
37+
38+
//--- B.swift
39+
public func funcB() {}
40+
41+
//--- client.swift
42+
import A
43+
import B
44+
import C

test/ScanDependencies/diagnose-missing-module-found-in-serialized-paths.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
// CHECK: error: Unable to find module dependency: 'C'
1313
// CHECK: note: a dependency of Swift module 'B': '{{.*}}B.swiftmodule'
1414
// CHECK: note: a dependency of main module 'deps'
15-
// CHECK: note: 'C' can be found on search path used to build module 'B': '{{.*}}moreDeps'. These search paths are not inherited by the current compilation.
15+
// CHECK: note: 'C' can be found on a search path used to build module 'B': '{{.*}}moreDeps'. These search paths are not inherited by the current compilation.
1616

1717
//--- moreDeps/C.swiftinterface
1818
// swift-interface-format-version: 1.0

0 commit comments

Comments
 (0)