[🍒][clang][scan-deps] Report a scanned TU's visible modules (#147969)

clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h

@@ -67,12 +67,19 @@ struct TranslationUnitDeps {
   /// determined that the differences are benign for this compilation.
   std::vector<ModuleID> ClangModuleDeps;
 
+  /// A list of module names that are visible to this translation unit. This
+  /// includes both direct and transitive module dependencies.
+  std::vector<std::string> VisibleModules;
+
   /// The CASID for input file dependency tree.
   std::optional<std::string> CASFileSystemRootID;
 
   /// The include-tree for input file dependency tree.
   std::optional<std::string> IncludeTreeID;
 
+  /// A list of the C++20 named modules this translation unit depends on.
+  std::vector<std::string> NamedModuleDeps;
+
   /// The sequence of commands required to build the translation unit. Commands
   /// should be executed in order.
   ///
@@ -192,7 +199,7 @@ class DependencyScanningTool {
   /// Given a compilation context specified via the Clang driver command-line,
   /// gather modular dependencies of module with the given name, and return the
   /// information needed for explicit build.
-  llvm::Expected<ModuleDepsGraph> getModuleDependencies(
+  llvm::Expected<TranslationUnitDeps> getModuleDependencies(
       StringRef ModuleName, const std::vector<std::string> &CommandLine,
       StringRef CWD, const llvm::DenseSet<ModuleID> &AlreadySeen,
       LookupModuleOutputCallback LookupModuleOutput);
@@ -255,6 +262,10 @@ class FullDependencyConsumer : public DependencyConsumer {
     DirectModuleDeps.push_back(ID);
   }
 
+  void handleVisibleModule(std::string ModuleName) override {
+    VisibleModules.push_back(ModuleName);
+  }
+
   void handleContextHash(std::string Hash) override {
     ContextHash = std::move(Hash);
   }
@@ -268,13 +279,13 @@ class FullDependencyConsumer : public DependencyConsumer {
   }
 
   TranslationUnitDeps takeTranslationUnitDeps();
-  ModuleDepsGraph takeModuleGraphDeps();
 
 private:
   std::vector<std::string> Dependencies;
   std::vector<PrebuiltModuleDep> PrebuiltModuleDeps;
   llvm::MapVector<ModuleID, ModuleDeps> ClangModuleDeps;
   std::vector<ModuleID> DirectModuleDeps;
+  std::vector<std::string> VisibleModules;
   std::vector<Command> Commands;
   std::string ContextHash;
   std::optional<std::string> CASFileSystemRootID;

clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h

@@ -66,6 +66,8 @@ class DependencyConsumer {
 
   virtual void handleDirectModuleDependency(ModuleID MD) = 0;
 
+  virtual void handleVisibleModule(std::string ModuleName) = 0;
+
   virtual void handleContextHash(std::string Hash) = 0;
 
   virtual void handleCASFileSystemRootID(std::string ID) {}

clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

@@ -300,6 +300,11 @@ class ModuleDepCollector final : public DependencyCollector {
   llvm::MapVector<const Module *, PrebuiltModuleDep> DirectPrebuiltModularDeps;
   /// Working set of direct modular dependencies.
   llvm::SetVector<const Module *> DirectModularDeps;
+  /// Working set of direct modular dependencies, as they were imported.
+  llvm::SmallPtrSet<const Module *, 32> DirectImports;
+  /// All direct and transitive visible modules.
+  llvm::StringSet<> VisibleModules;
+
   /// Options that control the dependency output generation.
   std::unique_ptr<DependencyOutputOptions> Opts;
   /// A Clang invocation that's based on the original TU invocation and that has
@@ -314,6 +319,9 @@ class ModuleDepCollector final : public DependencyCollector {
   /// Checks whether the module is known as being prebuilt.
   bool isPrebuiltModule(const Module *M);
 
+  /// Computes all visible modules resolved from direct imports.
+  void addVisibleModules();
+
   /// Adds \p Path to \c FileDeps, making it absolute if necessary.
   void addFileDep(StringRef Path);
   /// Adds \p Path to \c MD.FileDeps, making it absolute if necessary.

clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp

@@ -45,6 +45,7 @@ class MakeDependencyPrinterConsumer : public DependencyConsumer {
   void handlePrebuiltModuleDependency(PrebuiltModuleDep PMD) override {}
   void handleModuleDependency(ModuleDeps MD) override {}
   void handleDirectModuleDependency(ModuleID ID) override {}
+  void handleVisibleModule(std::string ModuleName) override {}
   void handleContextHash(std::string Hash) override {}
 
   void printDependencies(std::string &S) {
@@ -99,6 +100,8 @@ class EmptyDependencyConsumer : public DependencyConsumer {
 
   void handleModuleDependency(ModuleDeps MD) override {}
 
+  void handleVisibleModule(std::string ModuleName) override {}
+
   void handleDirectModuleDependency(ModuleID ID) override {}
 
   void handleContextHash(std::string Hash) override {}
@@ -279,7 +282,8 @@ DependencyScanningTool::getTranslationUnitDependencies(
   return Consumer.takeTranslationUnitDeps();
 }
 
-llvm::Expected<ModuleDepsGraph> DependencyScanningTool::getModuleDependencies(
+llvm::Expected<TranslationUnitDeps>
+DependencyScanningTool::getModuleDependencies(
     StringRef ModuleName, const std::vector<std::string> &CommandLine,
     StringRef CWD, const llvm::DenseSet<ModuleID> &AlreadySeen,
     LookupModuleOutputCallback LookupModuleOutput) {
@@ -289,7 +293,7 @@ llvm::Expected<ModuleDepsGraph> DependencyScanningTool::getModuleDependencies(
                                                   *Controller, ModuleName);
   if (Result)
     return std::move(Result);
-  return Consumer.takeModuleGraphDeps();
+  return Consumer.takeTranslationUnitDeps();
 }
 
 TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() {
@@ -298,6 +302,7 @@ TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() {
   TU.ID.ContextHash = std::move(ContextHash);
   TU.FileDeps = std::move(Dependencies);
   TU.PrebuiltModuleDeps = std::move(PrebuiltModuleDeps);
+  TU.VisibleModules = std::move(VisibleModules);
   TU.Commands = std::move(Commands);
   TU.CASFileSystemRootID = std::move(CASFileSystemRootID);
   TU.IncludeTreeID = std::move(IncludeTreeID);
@@ -315,21 +320,6 @@ TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() {
   return TU;
 }
 
-ModuleDepsGraph FullDependencyConsumer::takeModuleGraphDeps() {
-  ModuleDepsGraph ModuleGraph;
-
-  for (auto &&M : ClangModuleDeps) {
-    auto &MD = M.second;
-    // TODO: Avoid handleModuleDependency even being called for modules
-    //   we've already seen.
-    if (AlreadySeen.count(M.first))
-      continue;
-    ModuleGraph.push_back(std::move(MD));
-  }
-
-  return ModuleGraph;
-}
-
 CallbackActionController::~CallbackActionController() {}
 
 std::unique_ptr<DependencyActionController>

clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp

@@ -746,8 +746,10 @@ void ModuleDepCollectorPP::handleImport(const Module *Imported) {
   if (MDC.isPrebuiltModule(TopLevelModule))
     MDC.DirectPrebuiltModularDeps.insert(
         {TopLevelModule, PrebuiltModuleDep{TopLevelModule}});
-  else
+  else {
     MDC.DirectModularDeps.insert(TopLevelModule);
+    MDC.DirectImports.insert(Imported);
+  }
 }
 
 void ModuleDepCollectorPP::EndOfMainFile() {
@@ -779,6 +781,8 @@ void ModuleDepCollectorPP::EndOfMainFile() {
     if (!MDC.isPrebuiltModule(M))
       MDC.DirectModularDeps.insert(M);
 
+  MDC.addVisibleModules();
+
   for (const Module *M : MDC.DirectModularDeps)
     handleTopLevelModule(M);
 
@@ -798,6 +802,9 @@ void ModuleDepCollectorPP::EndOfMainFile() {
       MDC.Consumer.handleDirectModuleDependency(MDC.ModularDeps[M]->ID);
   }
 
+  for (auto &&I : MDC.VisibleModules)
+    MDC.Consumer.handleVisibleModule(std::string(I.getKey()));
+
   for (auto &&I : MDC.FileDeps)
     MDC.Consumer.handleFileDependency(I);
 
@@ -1100,6 +1107,29 @@ bool ModuleDepCollector::isPrebuiltModule(const Module *M) {
   return true;
 }
 
+void ModuleDepCollector::addVisibleModules() {
+  llvm::DenseSet<const Module *> ImportedModules;
+  auto InsertVisibleModules = [&](const Module *M) {
+    if (ImportedModules.contains(M))
+      return;
+
+    VisibleModules.insert(M->getTopLevelModuleName());
+    SmallVector<Module *> Stack;
+    M->getExportedModules(Stack);
+    while (!Stack.empty()) {
+      const Module *CurrModule = Stack.pop_back_val();
+      if (ImportedModules.contains(CurrModule))
+        continue;
+      ImportedModules.insert(CurrModule);
+      VisibleModules.insert(CurrModule->getTopLevelModuleName());
+      CurrModule->getExportedModules(Stack);
+    }
+  };
+
+  for (const Module *Import : DirectImports)
+    InsertVisibleModules(Import);
+}
+
 static StringRef makeAbsoluteAndPreferred(CompilerInstance &CI, StringRef Path,
                                           SmallVectorImpl<char> &Storage) {
   if (llvm::sys::path::is_absolute(Path) &&

clang/test/ClangScanDeps/visible-modules.c

@@ -0,0 +1,116 @@
+// This test verifies that the modules visible to the translation unit are computed in dependency scanning.
+// "client" in the first scan represents the translation unit that imports an explicit submodule, 
+//    that only exports one other module. 
+// In the second scan, the translation unit that imports an explicit submodule, 
+//    that exports an additional module. 
+// Thus, the dependencies of the top level module for the submodule always differ from what is visible to the TU.
+
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed -e "s|DIR|%/t|g" %t/compile-commands.json.in > %t/compile-commands.json
+// RUN: clang-scan-deps -emit-visible-modules -compilation-database %t/compile-commands.json \
+// RUN:   -j 1 -format experimental-full 2>&1 > %t/result-first-scan.json
+// RUN: cat %t/result-first-scan.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t --check-prefix=SINGLE
+
+/// Re-run scan with different module map for direct dependency.
+// RUN: mv %t/A_with_visible_export.modulemap %t/Sysroot/usr/include/A/module.modulemap
+// RUN: clang-scan-deps -emit-visible-modules -compilation-database %t/compile-commands.json \
+// RUN:   -j 1 -format experimental-full 2>&1 > %t/result.json
+// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t --check-prefix=MULTIPLE
+
+// RUN: %deps-to-rsp %t/result.json --module-name=transitive > %t/transitive.rsp
+// RUN: %deps-to-rsp %t/result.json --module-name=visible > %t/visible.rsp
+// RUN: %deps-to-rsp %t/result.json --module-name=invisible > %t/invisible.rsp
+// RUN: %deps-to-rsp %t/result.json --module-name=A > %t/A.rsp
+// RUN: %deps-to-rsp %t/result.json --tu-index=0 > %t/tu.rsp
+
+// RUN: %clang @%t/transitive.rsp
+// RUN: %clang @%t/visible.rsp
+// RUN: %clang @%t/invisible.rsp
+// RUN: %clang @%t/A.rsp
+
+/// Verify compilation & scan agree with each other.
+// RUN: not %clang @%t/tu.rsp 2>&1 | FileCheck %s --check-prefix=COMPILE
+
+// SINGLE:        "visible-clang-modules": [
+// SINGLE-NEXT:     "A"
+// SINGLE-NEXT:   ]
+
+// MULTIPLE:        "visible-clang-modules": [
+// MULTIPLE-NEXT:     "A",
+// MULTIPLE-NEXT:     "visible"
+// MULTIPLE-NEXT:   ]
+
+// COMPILE-NOT:   'visible_t' must be declared before it is used
+// COMPILE:       'transitive_t' must be declared before it is used
+// COMPILE:       'invisible_t' must be declared before it is used
+
+//--- compile-commands.json.in
+[
+{
+  "directory": "DIR",
+  "command": "clang -c DIR/client.c -isysroot DIR/Sysroot -IDIR/Sysroot/usr/include -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps",
+  "file": "DIR/client.c"
+}
+]
+
+//--- Sysroot/usr/include/A/module.modulemap
+module A {
+  explicit module visibleToTU {
+    header "visibleToTU.h"
+  }
+  explicit module invisibleToTU {
+    header "invisibleToTU.h" 
+  }
+}
+
+//--- A_with_visible_export.modulemap
+module A {
+  explicit module visibleToTU {
+    header "visibleToTU.h"
+    export visible
+  }
+  explicit module invisibleToTU {
+    header "invisibleToTU.h" 
+  }
+}
+
+//--- Sysroot/usr/include/A/visibleToTU.h
+#include <visible/visible.h>
+typedef int A_visibleToTU;
+
+//--- Sysroot/usr/include/A/invisibleToTU.h
+#include <invisible/invisible.h>
+typedef int A_invisibleToTU;
+
+//--- Sysroot/usr/include/invisible/module.modulemap
+module invisible {
+  umbrella "."
+}
+
+//--- Sysroot/usr/include/invisible/invisible.h
+typedef int invisible_t;
+
+//--- Sysroot/usr/include/visible/module.modulemap
+module visible {
+  umbrella "."
+}
+
+//--- Sysroot/usr/include/visible/visible.h
+#include <transitive/transitive.h>
+typedef int visible_t;
+
+//--- Sysroot/usr/include/transitive/module.modulemap
+module transitive {
+  umbrella "."
+}
+
+//--- Sysroot/usr/include/transitive/transitive.h
+typedef int transitive_t;
+
+//--- client.c
+#include <A/visibleToTU.h> 
+visible_t foo_v(void);
+// Both decls are not visible, thus should fail to actually compile.
+transitive_t foo_t(void);
+invisible_t foo_i(void);