Skip to content

Commit 55de9f8

Browse files
cyndyishidajansvoboda11
cyndyishida
and
jansvoboda11
committed
[clang][scan-deps] Report a scanned TU's visible modules (llvm#147969)
Clients of the dependency scanning service may need to add dependencies based on the visibility of importing modules, for example, when determining whether a Swift overlay dependency should be brought in based on whether there's a corresponding **visible** clang module for it. This patch introduces a new field `VisibleModules` that contains all the visible top-level modules in a given TU. Because visibility is determined by which headers or (sub)modules were imported, and not top-level module dependencies, the scanner now performs a separate DFS starting from what was directly imported for this computation. In my local performance testing, there was no observable performance impact. resolves: rdar://151416358 --------- Co-authored-by: Jan Svoboda <[email protected]> (cherry picked from commit 15c3793)
1 parent 4858864 commit 55de9f8

File tree

8 files changed

+213
-26
lines changed

8 files changed

+213
-26
lines changed

‎clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,12 +67,20 @@ struct TranslationUnitDeps {
6767
/// determined that the differences are benign for this compilation.
6868
std::vector<ModuleID> ClangModuleDeps;
6969

70+
/// A list of module names that are visible to this translation unit. This
71+
/// includes both direct and transitive module dependencies.
72+
std::vector<std::string> VisibleModules;
73+
7074
/// The CASID for input file dependency tree.
7175
std::optional<std::string> CASFileSystemRootID;
7276

7377
/// The include-tree for input file dependency tree.
7478
std::optional<std::string> IncludeTreeID;
7579

80+
/// A list of the C++20 named modules this translation unit depends on.
81+
std::vector<std::string> NamedModuleDeps;
82+
>>>>>>> 15c3793cdf94 ([clang][scan-deps] Report a scanned TU's visible modules (#147969))
83+
7684
/// The sequence of commands required to build the translation unit. Commands
7785
/// should be executed in order.
7886
///
@@ -192,7 +200,7 @@ class DependencyScanningTool {
192200
/// Given a compilation context specified via the Clang driver command-line,
193201
/// gather modular dependencies of module with the given name, and return the
194202
/// information needed for explicit build.
195-
llvm::Expected<ModuleDepsGraph> getModuleDependencies(
203+
llvm::Expected<TranslationUnitDeps> getModuleDependencies(
196204
StringRef ModuleName, const std::vector<std::string> &CommandLine,
197205
StringRef CWD, const llvm::DenseSet<ModuleID> &AlreadySeen,
198206
LookupModuleOutputCallback LookupModuleOutput);
@@ -255,6 +263,10 @@ class FullDependencyConsumer : public DependencyConsumer {
255263
DirectModuleDeps.push_back(ID);
256264
}
257265
266+
void handleVisibleModule(std::string ModuleName) override {
267+
VisibleModules.push_back(ModuleName);
268+
}
269+
258270
void handleContextHash(std::string Hash) override {
259271
ContextHash = std::move(Hash);
260272
}
@@ -268,13 +280,13 @@ class FullDependencyConsumer : public DependencyConsumer {
268280
}
269281
270282
TranslationUnitDeps takeTranslationUnitDeps();
271-
ModuleDepsGraph takeModuleGraphDeps();
272283
273284
private:
274285
std::vector<std::string> Dependencies;
275286
std::vector<PrebuiltModuleDep> PrebuiltModuleDeps;
276287
llvm::MapVector<ModuleID, ModuleDeps> ClangModuleDeps;
277288
std::vector<ModuleID> DirectModuleDeps;
289+
std::vector<std::string> VisibleModules;
278290
std::vector<Command> Commands;
279291
std::string ContextHash;
280292
std::optional<std::string> CASFileSystemRootID;

‎clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ class DependencyConsumer {
6666

6767
virtual void handleDirectModuleDependency(ModuleID MD) = 0;
6868

69+
virtual void handleVisibleModule(std::string ModuleName) = 0;
70+
6971
virtual void handleContextHash(std::string Hash) = 0;
7072

7173
virtual void handleCASFileSystemRootID(std::string ID) {}

‎clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,11 @@ class ModuleDepCollector final : public DependencyCollector {
300300
llvm::MapVector<const Module *, PrebuiltModuleDep> DirectPrebuiltModularDeps;
301301
/// Working set of direct modular dependencies.
302302
llvm::SetVector<const Module *> DirectModularDeps;
303+
/// Working set of direct modular dependencies, as they were imported.
304+
llvm::SmallPtrSet<const Module *, 32> DirectImports;
305+
/// All direct and transitive visible modules.
306+
llvm::StringSet<> VisibleModules;
307+
303308
/// Options that control the dependency output generation.
304309
std::unique_ptr<DependencyOutputOptions> Opts;
305310
/// A Clang invocation that's based on the original TU invocation and that has
@@ -314,6 +319,9 @@ class ModuleDepCollector final : public DependencyCollector {
314319
/// Checks whether the module is known as being prebuilt.
315320
bool isPrebuiltModule(const Module *M);
316321

322+
/// Computes all visible modules resolved from direct imports.
323+
void addVisibleModules();
324+
317325
/// Adds \p Path to \c FileDeps, making it absolute if necessary.
318326
void addFileDep(StringRef Path);
319327
/// Adds \p Path to \c MD.FileDeps, making it absolute if necessary.

‎clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp

Lines changed: 7 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ class MakeDependencyPrinterConsumer : public DependencyConsumer {
4545
void handlePrebuiltModuleDependency(PrebuiltModuleDep PMD) override {}
4646
void handleModuleDependency(ModuleDeps MD) override {}
4747
void handleDirectModuleDependency(ModuleID ID) override {}
48+
void handleVisibleModule(std::string ModuleName) override {}
4849
void handleContextHash(std::string Hash) override {}
4950

5051
void printDependencies(std::string &S) {
@@ -99,6 +100,8 @@ class EmptyDependencyConsumer : public DependencyConsumer {
99100

100101
void handleModuleDependency(ModuleDeps MD) override {}
101102

103+
void handleVisibleModule(std::string ModuleName) override {}
104+
102105
void handleDirectModuleDependency(ModuleID ID) override {}
103106

104107
void handleContextHash(std::string Hash) override {}
@@ -279,7 +282,8 @@ DependencyScanningTool::getTranslationUnitDependencies(
279282
return Consumer.takeTranslationUnitDeps();
280283
}
281284

282-
llvm::Expected<ModuleDepsGraph> DependencyScanningTool::getModuleDependencies(
285+
llvm::Expected<TranslationUnitDeps>
286+
DependencyScanningTool::getModuleDependencies(
283287
StringRef ModuleName, const std::vector<std::string> &CommandLine,
284288
StringRef CWD, const llvm::DenseSet<ModuleID> &AlreadySeen,
285289
LookupModuleOutputCallback LookupModuleOutput) {
@@ -289,7 +293,7 @@ llvm::Expected<ModuleDepsGraph> DependencyScanningTool::getModuleDependencies(
289293
*Controller, ModuleName);
290294
if (Result)
291295
return std::move(Result);
292-
return Consumer.takeModuleGraphDeps();
296+
return Consumer.takeTranslationUnitDeps();
293297
}
294298

295299
TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() {
@@ -298,6 +302,7 @@ TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() {
298302
TU.ID.ContextHash = std::move(ContextHash);
299303
TU.FileDeps = std::move(Dependencies);
300304
TU.PrebuiltModuleDeps = std::move(PrebuiltModuleDeps);
305+
TU.VisibleModules = std::move(VisibleModules);
301306
TU.Commands = std::move(Commands);
302307
TU.CASFileSystemRootID = std::move(CASFileSystemRootID);
303308
TU.IncludeTreeID = std::move(IncludeTreeID);
@@ -315,21 +320,6 @@ TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() {
315320
return TU;
316321
}
317322

318-
ModuleDepsGraph FullDependencyConsumer::takeModuleGraphDeps() {
319-
ModuleDepsGraph ModuleGraph;
320-
321-
for (auto &&M : ClangModuleDeps) {
322-
auto &MD = M.second;
323-
// TODO: Avoid handleModuleDependency even being called for modules
324-
// we've already seen.
325-
if (AlreadySeen.count(M.first))
326-
continue;
327-
ModuleGraph.push_back(std::move(MD));
328-
}
329-
330-
return ModuleGraph;
331-
}
332-
333323
CallbackActionController::~CallbackActionController() {}
334324

335325
std::unique_ptr<DependencyActionController>

‎clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -746,8 +746,10 @@ void ModuleDepCollectorPP::handleImport(const Module *Imported) {
746746
if (MDC.isPrebuiltModule(TopLevelModule))
747747
MDC.DirectPrebuiltModularDeps.insert(
748748
{TopLevelModule, PrebuiltModuleDep{TopLevelModule}});
749-
else
749+
else {
750750
MDC.DirectModularDeps.insert(TopLevelModule);
751+
MDC.DirectImports.insert(Imported);
752+
}
751753
}
752754

753755
void ModuleDepCollectorPP::EndOfMainFile() {
@@ -779,6 +781,8 @@ void ModuleDepCollectorPP::EndOfMainFile() {
779781
if (!MDC.isPrebuiltModule(M))
780782
MDC.DirectModularDeps.insert(M);
781783

784+
MDC.addVisibleModules();
785+
782786
for (const Module *M : MDC.DirectModularDeps)
783787
handleTopLevelModule(M);
784788

@@ -798,6 +802,9 @@ void ModuleDepCollectorPP::EndOfMainFile() {
798802
MDC.Consumer.handleDirectModuleDependency(MDC.ModularDeps[M]->ID);
799803
}
800804

805+
for (auto &&I : MDC.VisibleModules)
806+
MDC.Consumer.handleVisibleModule(std::string(I.getKey()));
807+
801808
for (auto &&I : MDC.FileDeps)
802809
MDC.Consumer.handleFileDependency(I);
803810

@@ -1100,6 +1107,29 @@ bool ModuleDepCollector::isPrebuiltModule(const Module *M) {
11001107
return true;
11011108
}
11021109

1110+
void ModuleDepCollector::addVisibleModules() {
1111+
llvm::DenseSet<const Module *> ImportedModules;
1112+
auto InsertVisibleModules = [&](const Module *M) {
1113+
if (ImportedModules.contains(M))
1114+
return;
1115+
1116+
VisibleModules.insert(M->getTopLevelModuleName());
1117+
SmallVector<Module *> Stack;
1118+
M->getExportedModules(Stack);
1119+
while (!Stack.empty()) {
1120+
const Module *CurrModule = Stack.pop_back_val();
1121+
if (ImportedModules.contains(CurrModule))
1122+
continue;
1123+
ImportedModules.insert(CurrModule);
1124+
VisibleModules.insert(CurrModule->getTopLevelModuleName());
1125+
CurrModule->getExportedModules(Stack);
1126+
}
1127+
};
1128+
1129+
for (const Module *Import : DirectImports)
1130+
InsertVisibleModules(Import);
1131+
}
1132+
11031133
static StringRef makeAbsoluteAndPreferred(CompilerInstance &CI, StringRef Path,
11041134
SmallVectorImpl<char> &Storage) {
11051135
if (llvm::sys::path::is_absolute(Path) &&

‎clang/test/ClangScanDeps/visible-modules.c

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
// This test verifies that the modules visible to the translation unit are computed in dependency scanning.
2+
// "client" in the first scan represents the translation unit that imports an explicit submodule,
3+
// that only exports one other module.
4+
// In the second scan, the translation unit that imports an explicit submodule,
5+
// that exports an additional module.
6+
// Thus, the dependencies of the top level module for the submodule always differ from what is visible to the TU.
7+
8+
// RUN: rm -rf %t
9+
// RUN: split-file %s %t
10+
// RUN: sed -e "s|DIR|%/t|g" %t/compile-commands.json.in > %t/compile-commands.json
11+
// RUN: clang-scan-deps -emit-visible-modules -compilation-database %t/compile-commands.json \
12+
// RUN: -j 1 -format experimental-full 2>&1 > %t/result-first-scan.json
13+
// RUN: cat %t/result-first-scan.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t --check-prefix=SINGLE
14+
15+
/// Re-run scan with different module map for direct dependency.
16+
// RUN: mv %t/A_with_visible_export.modulemap %t/Sysroot/usr/include/A/module.modulemap
17+
// RUN: clang-scan-deps -emit-visible-modules -compilation-database %t/compile-commands.json \
18+
// RUN: -j 1 -format experimental-full 2>&1 > %t/result.json
19+
// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t --check-prefix=MULTIPLE
20+
21+
// RUN: %deps-to-rsp %t/result.json --module-name=transitive > %t/transitive.rsp
22+
// RUN: %deps-to-rsp %t/result.json --module-name=visible > %t/visible.rsp
23+
// RUN: %deps-to-rsp %t/result.json --module-name=invisible > %t/invisible.rsp
24+
// RUN: %deps-to-rsp %t/result.json --module-name=A > %t/A.rsp
25+
// RUN: %deps-to-rsp %t/result.json --tu-index=0 > %t/tu.rsp
26+
27+
// RUN: %clang @%t/transitive.rsp
28+
// RUN: %clang @%t/visible.rsp
29+
// RUN: %clang @%t/invisible.rsp
30+
// RUN: %clang @%t/A.rsp
31+
32+
/// Verify compilation & scan agree with each other.
33+
// RUN: not %clang @%t/tu.rsp 2>&1 | FileCheck %s --check-prefix=COMPILE
34+
35+
// SINGLE: "visible-clang-modules": [
36+
// SINGLE-NEXT: "A"
37+
// SINGLE-NEXT: ]
38+
39+
// MULTIPLE: "visible-clang-modules": [
40+
// MULTIPLE-NEXT: "A",
41+
// MULTIPLE-NEXT: "visible"
42+
// MULTIPLE-NEXT: ]
43+
44+
// COMPILE-NOT: 'visible_t' must be declared before it is used
45+
// COMPILE: 'transitive_t' must be declared before it is used
46+
// COMPILE: 'invisible_t' must be declared before it is used
47+
48+
//--- compile-commands.json.in
49+
[
50+
{
51+
"directory": "DIR",
52+
"command": "clang -c DIR/client.c -isysroot DIR/Sysroot -IDIR/Sysroot/usr/include -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps",
53+
"file": "DIR/client.c"
54+
}
55+
]
56+
57+
//--- Sysroot/usr/include/A/module.modulemap
58+
module A {
59+
explicit module visibleToTU {
60+
header "visibleToTU.h"
61+
}
62+
explicit module invisibleToTU {
63+
header "invisibleToTU.h"
64+
}
65+
}
66+
67+
//--- A_with_visible_export.modulemap
68+
module A {
69+
explicit module visibleToTU {
70+
header "visibleToTU.h"
71+
export visible
72+
}
73+
explicit module invisibleToTU {
74+
header "invisibleToTU.h"
75+
}
76+
}
77+
78+
//--- Sysroot/usr/include/A/visibleToTU.h
79+
#include <visible/visible.h>
80+
typedef int A_visibleToTU;
81+
82+
//--- Sysroot/usr/include/A/invisibleToTU.h
83+
#include <invisible/invisible.h>
84+
typedef int A_invisibleToTU;
85+
86+
//--- Sysroot/usr/include/invisible/module.modulemap
87+
module invisible {
88+
umbrella "."
89+
}
90+
91+
//--- Sysroot/usr/include/invisible/invisible.h
92+
typedef int invisible_t;
93+
94+
//--- Sysroot/usr/include/visible/module.modulemap
95+
module visible {
96+
umbrella "."
97+
}
98+
99+
//--- Sysroot/usr/include/visible/visible.h
100+
#include <transitive/transitive.h>
101+
typedef int visible_t;
102+
103+
//--- Sysroot/usr/include/transitive/module.modulemap
104+
module transitive {
105+
umbrella "."
106+
}
107+
108+
//--- Sysroot/usr/include/transitive/transitive.h
109+
typedef int transitive_t;
110+
111+
//--- client.c
112+
#include <A/visibleToTU.h>
113+
visible_t foo_v(void);
114+
// Both decls are not visible, thus should fail to actually compile.
115+
transitive_t foo_t(void);
116+
invisible_t foo_i(void);

0 commit comments

Comments
 (0)