diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h index be83bd47394bf..d4911e694f661 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h @@ -67,12 +67,19 @@ struct TranslationUnitDeps { /// determined that the differences are benign for this compilation. std::vector ClangModuleDeps; + /// A list of module names that are visible to this translation unit. This + /// includes both direct and transitive module dependencies. + std::vector VisibleModules; + /// The CASID for input file dependency tree. std::optional CASFileSystemRootID; /// The include-tree for input file dependency tree. std::optional IncludeTreeID; + /// A list of the C++20 named modules this translation unit depends on. + std::vector NamedModuleDeps; + /// The sequence of commands required to build the translation unit. Commands /// should be executed in order. /// @@ -192,7 +199,7 @@ class DependencyScanningTool { /// Given a compilation context specified via the Clang driver command-line, /// gather modular dependencies of module with the given name, and return the /// information needed for explicit build. - llvm::Expected getModuleDependencies( + llvm::Expected getModuleDependencies( StringRef ModuleName, const std::vector &CommandLine, StringRef CWD, const llvm::DenseSet &AlreadySeen, LookupModuleOutputCallback LookupModuleOutput); @@ -255,6 +262,10 @@ class FullDependencyConsumer : public DependencyConsumer { DirectModuleDeps.push_back(ID); } + void handleVisibleModule(std::string ModuleName) override { + VisibleModules.push_back(ModuleName); + } + void handleContextHash(std::string Hash) override { ContextHash = std::move(Hash); } @@ -268,13 +279,13 @@ class FullDependencyConsumer : public DependencyConsumer { } TranslationUnitDeps takeTranslationUnitDeps(); - ModuleDepsGraph takeModuleGraphDeps(); private: std::vector Dependencies; std::vector PrebuiltModuleDeps; llvm::MapVector ClangModuleDeps; std::vector DirectModuleDeps; + std::vector VisibleModules; std::vector Commands; std::string ContextHash; std::optional CASFileSystemRootID; diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h index 9194d7726cf39..ab85ce5c1c9dd 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h @@ -66,6 +66,8 @@ class DependencyConsumer { virtual void handleDirectModuleDependency(ModuleID MD) = 0; + virtual void handleVisibleModule(std::string ModuleName) = 0; + virtual void handleContextHash(std::string Hash) = 0; virtual void handleCASFileSystemRootID(std::string ID) {} diff --git a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h index 814e794727996..29b4dfcb7c820 100644 --- a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h +++ b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h @@ -300,6 +300,11 @@ class ModuleDepCollector final : public DependencyCollector { llvm::MapVector DirectPrebuiltModularDeps; /// Working set of direct modular dependencies. llvm::SetVector DirectModularDeps; + /// Working set of direct modular dependencies, as they were imported. + llvm::SmallPtrSet DirectImports; + /// All direct and transitive visible modules. + llvm::StringSet<> VisibleModules; + /// Options that control the dependency output generation. std::unique_ptr Opts; /// A Clang invocation that's based on the original TU invocation and that has @@ -314,6 +319,9 @@ class ModuleDepCollector final : public DependencyCollector { /// Checks whether the module is known as being prebuilt. bool isPrebuiltModule(const Module *M); + /// Computes all visible modules resolved from direct imports. + void addVisibleModules(); + /// Adds \p Path to \c FileDeps, making it absolute if necessary. void addFileDep(StringRef Path); /// Adds \p Path to \c MD.FileDeps, making it absolute if necessary. diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp index e158b89aa21e6..af3be830081dd 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp @@ -45,6 +45,7 @@ class MakeDependencyPrinterConsumer : public DependencyConsumer { void handlePrebuiltModuleDependency(PrebuiltModuleDep PMD) override {} void handleModuleDependency(ModuleDeps MD) override {} void handleDirectModuleDependency(ModuleID ID) override {} + void handleVisibleModule(std::string ModuleName) override {} void handleContextHash(std::string Hash) override {} void printDependencies(std::string &S) { @@ -99,6 +100,8 @@ class EmptyDependencyConsumer : public DependencyConsumer { void handleModuleDependency(ModuleDeps MD) override {} + void handleVisibleModule(std::string ModuleName) override {} + void handleDirectModuleDependency(ModuleID ID) override {} void handleContextHash(std::string Hash) override {} @@ -279,7 +282,8 @@ DependencyScanningTool::getTranslationUnitDependencies( return Consumer.takeTranslationUnitDeps(); } -llvm::Expected DependencyScanningTool::getModuleDependencies( +llvm::Expected +DependencyScanningTool::getModuleDependencies( StringRef ModuleName, const std::vector &CommandLine, StringRef CWD, const llvm::DenseSet &AlreadySeen, LookupModuleOutputCallback LookupModuleOutput) { @@ -289,7 +293,7 @@ llvm::Expected DependencyScanningTool::getModuleDependencies( *Controller, ModuleName); if (Result) return std::move(Result); - return Consumer.takeModuleGraphDeps(); + return Consumer.takeTranslationUnitDeps(); } TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() { @@ -298,6 +302,7 @@ TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() { TU.ID.ContextHash = std::move(ContextHash); TU.FileDeps = std::move(Dependencies); TU.PrebuiltModuleDeps = std::move(PrebuiltModuleDeps); + TU.VisibleModules = std::move(VisibleModules); TU.Commands = std::move(Commands); TU.CASFileSystemRootID = std::move(CASFileSystemRootID); TU.IncludeTreeID = std::move(IncludeTreeID); @@ -315,21 +320,6 @@ TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() { return TU; } -ModuleDepsGraph FullDependencyConsumer::takeModuleGraphDeps() { - ModuleDepsGraph ModuleGraph; - - for (auto &&M : ClangModuleDeps) { - auto &MD = M.second; - // TODO: Avoid handleModuleDependency even being called for modules - // we've already seen. - if (AlreadySeen.count(M.first)) - continue; - ModuleGraph.push_back(std::move(MD)); - } - - return ModuleGraph; -} - CallbackActionController::~CallbackActionController() {} std::unique_ptr diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index a613981a7d0c8..30fb5279b5bae 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -746,8 +746,10 @@ void ModuleDepCollectorPP::handleImport(const Module *Imported) { if (MDC.isPrebuiltModule(TopLevelModule)) MDC.DirectPrebuiltModularDeps.insert( {TopLevelModule, PrebuiltModuleDep{TopLevelModule}}); - else + else { MDC.DirectModularDeps.insert(TopLevelModule); + MDC.DirectImports.insert(Imported); + } } void ModuleDepCollectorPP::EndOfMainFile() { @@ -779,6 +781,8 @@ void ModuleDepCollectorPP::EndOfMainFile() { if (!MDC.isPrebuiltModule(M)) MDC.DirectModularDeps.insert(M); + MDC.addVisibleModules(); + for (const Module *M : MDC.DirectModularDeps) handleTopLevelModule(M); @@ -798,6 +802,9 @@ void ModuleDepCollectorPP::EndOfMainFile() { MDC.Consumer.handleDirectModuleDependency(MDC.ModularDeps[M]->ID); } + for (auto &&I : MDC.VisibleModules) + MDC.Consumer.handleVisibleModule(std::string(I.getKey())); + for (auto &&I : MDC.FileDeps) MDC.Consumer.handleFileDependency(I); @@ -1100,6 +1107,29 @@ bool ModuleDepCollector::isPrebuiltModule(const Module *M) { return true; } +void ModuleDepCollector::addVisibleModules() { + llvm::DenseSet ImportedModules; + auto InsertVisibleModules = [&](const Module *M) { + if (ImportedModules.contains(M)) + return; + + VisibleModules.insert(M->getTopLevelModuleName()); + SmallVector Stack; + M->getExportedModules(Stack); + while (!Stack.empty()) { + const Module *CurrModule = Stack.pop_back_val(); + if (ImportedModules.contains(CurrModule)) + continue; + ImportedModules.insert(CurrModule); + VisibleModules.insert(CurrModule->getTopLevelModuleName()); + CurrModule->getExportedModules(Stack); + } + }; + + for (const Module *Import : DirectImports) + InsertVisibleModules(Import); +} + static StringRef makeAbsoluteAndPreferred(CompilerInstance &CI, StringRef Path, SmallVectorImpl &Storage) { if (llvm::sys::path::is_absolute(Path) && diff --git a/clang/test/ClangScanDeps/visible-modules.c b/clang/test/ClangScanDeps/visible-modules.c new file mode 100644 index 0000000000000..77716a4956f00 --- /dev/null +++ b/clang/test/ClangScanDeps/visible-modules.c @@ -0,0 +1,116 @@ +// This test verifies that the modules visible to the translation unit are computed in dependency scanning. +// "client" in the first scan represents the translation unit that imports an explicit submodule, +// that only exports one other module. +// In the second scan, the translation unit that imports an explicit submodule, +// that exports an additional module. +// Thus, the dependencies of the top level module for the submodule always differ from what is visible to the TU. + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed -e "s|DIR|%/t|g" %t/compile-commands.json.in > %t/compile-commands.json +// RUN: clang-scan-deps -emit-visible-modules -compilation-database %t/compile-commands.json \ +// RUN: -j 1 -format experimental-full 2>&1 > %t/result-first-scan.json +// RUN: cat %t/result-first-scan.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t --check-prefix=SINGLE + +/// Re-run scan with different module map for direct dependency. +// RUN: mv %t/A_with_visible_export.modulemap %t/Sysroot/usr/include/A/module.modulemap +// RUN: clang-scan-deps -emit-visible-modules -compilation-database %t/compile-commands.json \ +// RUN: -j 1 -format experimental-full 2>&1 > %t/result.json +// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t --check-prefix=MULTIPLE + +// RUN: %deps-to-rsp %t/result.json --module-name=transitive > %t/transitive.rsp +// RUN: %deps-to-rsp %t/result.json --module-name=visible > %t/visible.rsp +// RUN: %deps-to-rsp %t/result.json --module-name=invisible > %t/invisible.rsp +// RUN: %deps-to-rsp %t/result.json --module-name=A > %t/A.rsp +// RUN: %deps-to-rsp %t/result.json --tu-index=0 > %t/tu.rsp + +// RUN: %clang @%t/transitive.rsp +// RUN: %clang @%t/visible.rsp +// RUN: %clang @%t/invisible.rsp +// RUN: %clang @%t/A.rsp + +/// Verify compilation & scan agree with each other. +// RUN: not %clang @%t/tu.rsp 2>&1 | FileCheck %s --check-prefix=COMPILE + +// SINGLE: "visible-clang-modules": [ +// SINGLE-NEXT: "A" +// SINGLE-NEXT: ] + +// MULTIPLE: "visible-clang-modules": [ +// MULTIPLE-NEXT: "A", +// MULTIPLE-NEXT: "visible" +// MULTIPLE-NEXT: ] + +// COMPILE-NOT: 'visible_t' must be declared before it is used +// COMPILE: 'transitive_t' must be declared before it is used +// COMPILE: 'invisible_t' must be declared before it is used + +//--- compile-commands.json.in +[ +{ + "directory": "DIR", + "command": "clang -c DIR/client.c -isysroot DIR/Sysroot -IDIR/Sysroot/usr/include -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps", + "file": "DIR/client.c" +} +] + +//--- Sysroot/usr/include/A/module.modulemap +module A { + explicit module visibleToTU { + header "visibleToTU.h" + } + explicit module invisibleToTU { + header "invisibleToTU.h" + } +} + +//--- A_with_visible_export.modulemap +module A { + explicit module visibleToTU { + header "visibleToTU.h" + export visible + } + explicit module invisibleToTU { + header "invisibleToTU.h" + } +} + +//--- Sysroot/usr/include/A/visibleToTU.h +#include +typedef int A_visibleToTU; + +//--- Sysroot/usr/include/A/invisibleToTU.h +#include +typedef int A_invisibleToTU; + +//--- Sysroot/usr/include/invisible/module.modulemap +module invisible { + umbrella "." +} + +//--- Sysroot/usr/include/invisible/invisible.h +typedef int invisible_t; + +//--- Sysroot/usr/include/visible/module.modulemap +module visible { + umbrella "." +} + +//--- Sysroot/usr/include/visible/visible.h +#include +typedef int visible_t; + +//--- Sysroot/usr/include/transitive/module.modulemap +module transitive { + umbrella "." +} + +//--- Sysroot/usr/include/transitive/transitive.h +typedef int transitive_t; + +//--- client.c +#include +visible_t foo_v(void); +// Both decls are not visible, thus should fail to actually compile. +transitive_t foo_t(void); +invisible_t foo_i(void); diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index cb108351431d9..5b50fab7183a0 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -98,6 +98,7 @@ static bool DeprecatedDriverCommand; static ResourceDirRecipeKind ResourceDirRecipe; static bool Verbose; static bool PrintTiming; +static bool EmitVisibleModules; static llvm::BumpPtrAllocator Alloc; static llvm::StringSaver Saver{Alloc}; static std::vector CommandLine; @@ -248,6 +249,8 @@ static void ParseArgs(int argc, char **argv) { PrintTiming = Args.hasArg(OPT_print_timing); + EmitVisibleModules = Args.hasArg(OPT_emit_visible_modules); + EmitCASCompDB = Args.hasArg(OPT_emit_cas_compdb); InMemoryCAS = Args.hasArg(OPT_in_memory_cas); @@ -631,6 +634,14 @@ static auto toJSONSorted(llvm::json::OStream &JOS, }; } +static auto toJSONSorted(llvm::json::OStream &JOS, std::vector V) { + llvm::sort(V); + return [&JOS, V = std::move(V)] { + for (const StringRef Entry : V) + JOS.value(Entry); + }; +} + // Thread safe. class FullDeps { public: @@ -647,6 +658,10 @@ class FullDeps { ID.ModuleDeps = std::move(TUDeps.ClangModuleDeps); ID.CASFileSystemRootID = std::move(TUDeps.CASFileSystemRootID); ID.IncludeTreeID = std::move(TUDeps.IncludeTreeID); + ID.NamedModule = std::move(TUDeps.ID.ModuleName); + ID.NamedModuleDeps = std::move(TUDeps.NamedModuleDeps); + ID.ClangModuleDeps = std::move(TUDeps.ClangModuleDeps); + ID.VisibleModules = std::move(TUDeps.VisibleModules); ID.DriverCommandLine = std::move(TUDeps.DriverCommandLine); ID.Commands = std::move(TUDeps.Commands); @@ -786,6 +801,9 @@ class FullDeps { JOS.attributeArray("file-deps", toJSONStrings(JOS, I.FileDeps)); JOS.attribute("input-file", StringRef(I.FileName)); + if (EmitVisibleModules) + JOS.attributeArray("visible-clang-modules", + toJSONSorted(JOS, I.VisibleModules)); }); } } else { @@ -806,6 +824,9 @@ class FullDeps { JOS.attributeArray("file-deps", toJSONStrings(JOS, I.FileDeps)); JOS.attribute("input-file", StringRef(I.FileName)); + if (EmitVisibleModules) + JOS.attributeArray("visible-clang-modules", + toJSONSorted(JOS, I.VisibleModules)); }); } }); @@ -857,6 +878,10 @@ class FullDeps { std::vector ModuleDeps; std::optional CASFileSystemRootID; std::optional IncludeTreeID; + std::string NamedModule; + std::vector NamedModuleDeps; + std::vector ClangModuleDeps; + std::vector VisibleModules; std::vector DriverCommandLine; std::vector Commands; }; @@ -884,11 +909,12 @@ static bool handleTranslationUnitResult( return false; } -static bool handleModuleResult( - StringRef ModuleName, llvm::Expected &MaybeModuleGraph, - FullDeps &FD, size_t InputIndex, SharedStream &OS, SharedStream &Errs) { - if (!MaybeModuleGraph) { - llvm::handleAllErrors(MaybeModuleGraph.takeError(), +static bool handleModuleResult(StringRef ModuleName, + llvm::Expected &MaybeTUDeps, + FullDeps &FD, size_t InputIndex, + SharedStream &OS, SharedStream &Errs) { + if (!MaybeTUDeps) { + llvm::handleAllErrors(MaybeTUDeps.takeError(), [&ModuleName, &Errs](llvm::StringError &Err) { Errs.applyLocked([&](raw_ostream &OS) { OS << "Error while scanning dependencies for " @@ -898,7 +924,7 @@ static bool handleModuleResult( }); return true; } - FD.mergeDeps(std::move(*MaybeModuleGraph), InputIndex); + FD.mergeDeps(std::move(MaybeTUDeps->ModuleGraph), InputIndex); return false; } diff --git a/clang/tools/clang-scan-deps/Opts.td b/clang/tools/clang-scan-deps/Opts.td index 7841f6e33f44e..4f12a697fc3b9 100644 --- a/clang/tools/clang-scan-deps/Opts.td +++ b/clang/tools/clang-scan-deps/Opts.td @@ -52,6 +52,9 @@ defm prefix_map : Eq<"prefix-map", "Path to remap, as \"=\".">; def print_timing : F<"print-timing", "Print timing information">; +def emit_visible_modules + : F<"emit-visible-modules", "emit visible modules in primary output">; + def verbose : F<"v", "Use verbose output">; def round_trip_args : F<"round-trip-args", "verify that command-line arguments are canonical by parsing and re-serializing">;