Author: Cyndy Ishida Date: 2026-10-30T20:46:28-07:00 New Revision: ff065cf32589d1d4b74b26b73f187f0c025cfbb6
URL: https://github.com/llvm/llvm-project/commit/ff065cf32589d1d4b74b26b73f187f0c025cfbb6 DIFF: https://github.com/llvm/llvm-project/commit/ff065cf32589d1d4b74b26b73f187f0c025cfbb6.diff LOG: [clang][DepScan] Introduce API to find modulemaps without loading (#235845) Introduce a way to batch look up the modulemap that defines each of a set of module names. `getModuleMapsForModules` builds a compiler instance from the same command line as a by-name scan, then does every lookup in a separate header search with lazy modulemap loading turned on and diagnostics ignored. * Modulemaps passed with `-fmodule-map-file` are parsed first, all other modules are resolved by by-name header search. The main use case is to let the Swift dependency scanner invalidate its serialized dependency graph without the cost of loading the referenced modules. resolves clang side of: rdar://190131044 Added: Modified: clang/include/clang/DependencyScanning/DependencyScanningWorker.h clang/include/clang/Tooling/DependencyScanningTool.h clang/lib/DependencyScanning/DependencyScanningWorker.cpp clang/lib/Tooling/DependencyScanningTool.cpp clang/unittests/Tooling/DependencyScannerTest.cpp Removed: ################################################################################ diff --git a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h index a7a9bcad7f81a8..af32a8677e4774 100644 --- a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h @@ -17,6 +17,7 @@ #include "clang/DependencyScanning/ModuleDepCollector.h" #include "clang/Frontend/PCHContainerOperations.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/ADT/StringMap.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBufferRef.h" #include "llvm/Support/VirtualFileSystem.h" @@ -77,6 +78,14 @@ class DependencyScanningWorker { llvm::function_ref<std::optional<std::string>()> getNextName, DependencyConsumer &DepConsumer); + /// Creates a compiler instance to return a map of module names to their + /// defined module map path. + std::optional<llvm::StringMap<std::string>> computeModuleMapsForModules( + StringRef CWD, ArrayRef<std::string> CC1CommandLine, + IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS, + DiagnosticConsumer &DiagConsumer, DependencyActionController &Controller, + ArrayRef<StringRef> ModuleNames); + /// Creates the effective VFS that will be used for the scan. /// /// If provided, OverlayFS will be overlaid on top of the Worker's dependency diff --git a/clang/include/clang/Tooling/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanningTool.h index 7b7325446b928e..43d1691e8f4c98 100644 --- a/clang/include/clang/Tooling/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanningTool.h @@ -118,6 +118,19 @@ class DependencyScanningTool { llvm::function_ref<std::optional<std::string>()> getNextName, dependencies::DependencyConsumer &DepConsumer); + /// Find the module maps that define each module in \p ModuleNames, without + /// loading or reporting diagnostics about the modules. + /// + /// \returns a map from each module name that is found to the path of the + /// module map that defines it, or std::nullopt if the compiler invocation + /// could not be set up. Names provided by a prebuilt module file, or with no + /// module map found, are omitted. + std::optional<llvm::StringMap<std::string>> + getModuleMapsForModules(StringRef CWD, ArrayRef<std::string> CommandLine, + DiagnosticConsumer &DiagConsumer, + dependencies::DependencyActionController &Controller, + ArrayRef<StringRef> ModuleNames); + /// Returns the worker tracing VFS, if it was requested via the service. llvm::vfs::TracingFileSystem *getWorkerTracingVFS() const { return Worker.getTracingVFS(); diff --git a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp index af9946a64feba1..410f38c333a81e 100644 --- a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp @@ -14,6 +14,7 @@ #include "clang/DependencyScanning/DependencyConsumer.h" #include "clang/DependencyScanning/DependencyScanningFilesystem.h" #include "clang/Frontend/FrontendActions.h" +#include "clang/Lex/HeaderSearch.h" #include "clang/Lex/Preprocessor.h" #include "clang/Sema/SemaOpenACC.h" #include "clang/Serialization/ObjectFilePCHContainerReader.h" @@ -702,6 +703,62 @@ createScanningDiagOptions(ArrayRef<std::string> CommandLine) { return DiagOpts; } +namespace { +/// Find the module maps that define modules without loading them. +class ModuleMapLookup { +public: + explicit ModuleMapLookup(CompilerInstance &CI) + : CI(CI), HSOpts(CI.getHeaderSearchOpts()), + Diags(DiagnosticIDs::create(), DiagOpts, new IgnoringDiagConsumer()), + SourceMgr(Diags, CI.getFileManager()), + HS(HSOpts, SourceMgr, Diags, CI.getLangOpts(), &CI.getTarget()) { + HSOpts.LazyLoadModuleMaps = true; + // Ignore verbose mode as it's already respected by the real scan. + HSOpts.Verbose = false; + ApplyHeaderSearchOptions(HS, HSOpts, CI.getLangOpts(), + CI.getTarget().getTriple()); + } + + llvm::StringMap<std::string> + computeModuleMapsForModules(ArrayRef<StringRef> ModuleNames) { + // Resolve any modulemaps passed directly first. + for (const std::string &Filename : CI.getFrontendOpts().ModuleMapFiles) + if (OptionalFileEntryRef File = + CI.getFileManager().getOptionalFileRef(Filename)) + HS.parseModuleMapFile(*File, /*IsSystem=*/false, + /*ImplicitlyDiscovered=*/false); + + const bool HasPrebuiltModules = !HSOpts.PrebuiltModuleFiles.empty() || + !HSOpts.PrebuiltModulePaths.empty(); + + llvm::StringMap<std::string> Result; + // Then resolve modulemaps through header search. + for (StringRef Name : ModuleNames) { + // Skip over prebuilt modules. + if (HasPrebuiltModules && !HS.getPrebuiltModuleFileName(Name).empty()) + continue; + OptionalFileEntryRef File = HS.lookupModuleMapFileForModule(Name); + // Skip over modules that didn't find a modulemap. + if (!File) + continue; + // Canonicalize the path the same way as for the real scan. + SmallString<128> Path = File->getNameAsRequested(); + HS.getModuleMap().canonicalizeModuleMapPath(Path); + Result[Name] = std::string(Path); + } + return Result; + } + +private: + CompilerInstance &CI; + HeaderSearchOptions HSOpts; + DiagnosticOptions DiagOpts; + DiagnosticsEngine Diags; + SourceManager SourceMgr; + HeaderSearch HS; +}; +} // namespace + class CompilerInstanceWithContext { // Context DependencyScanningWorker &Worker; @@ -827,6 +884,24 @@ class CompilerInstanceWithContext { return std::move(CIWC); } + /// Initialize from a cc1 command line, reporting diagnostics to + /// \p DiagConsumer. + static std::optional<CompilerInstanceWithContext> + initializeFromCC1Commandline( + DependencyScanningWorker &Worker, StringRef CWD, + ArrayRef<std::string> CC1CommandLine, DiagnosticConsumer &DiagConsumer, + IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS, + DependencyActionController &Controller) { + auto FS = Worker.makeEffectiveVFS(CWD, OverlayFS); + auto DiagOpts = createScanningDiagOptions(CC1CommandLine); + auto DiagEngine = + CompilerInstance::createDiagnostics(*FS, *DiagOpts, &DiagConsumer, + /*ShouldOwnClient=*/false); + return initializeFromCC1Commandline(Worker, CWD, CC1CommandLine, + *DiagEngine, std::move(OverlayFS), + Controller); + } + bool computeDependencies(StringRef ModuleName, DependencyConsumer &Consumer, DependencyActionController &Controller) { Worker.Service.getLogger().log() << "start scan_by_name: " << ModuleName; @@ -939,6 +1014,12 @@ class CompilerInstanceWithContext { return true; } + llvm::StringMap<std::string> + computeModuleMapsForModules(ArrayRef<StringRef> ModuleNames) { + assert(CIPtr && "CIPtr must be initialized before calling this method"); + return ModuleMapLookup(*CIPtr).computeModuleMapsForModules(ModuleNames); + } + std::shared_ptr<ModuleDepCollector> scanTranslationUnit(DependencyConsumer &Consumer, DependencyActionController &Controller) { @@ -1105,14 +1186,9 @@ bool DependencyScanningWorker::computeDependenciesByName( DiagnosticConsumer &DiagConsumer, DependencyActionController &Controller, llvm::function_ref<std::optional<std::string>()> getNextName, DependencyConsumer &DepConsumer) { - auto FS = makeEffectiveVFS(CWD, OverlayFS); - auto DiagOpts = createScanningDiagOptions(CC1CommandLine); - auto DiagEngine = - CompilerInstance::createDiagnostics(*FS, *DiagOpts, &DiagConsumer, - /*ShouldOwnClient=*/false); std::optional<CompilerInstanceWithContext> CIWC = CompilerInstanceWithContext::initializeFromCC1Commandline( - *this, CWD, CC1CommandLine, *DiagEngine, std::move(OverlayFS), + *this, CWD, CC1CommandLine, DiagConsumer, std::move(OverlayFS), Controller); if (!CIWC) return false; @@ -1126,3 +1202,18 @@ bool DependencyScanningWorker::computeDependenciesByName( } return AllScansSucceeded; } + +std::optional<llvm::StringMap<std::string>> +DependencyScanningWorker::computeModuleMapsForModules( + StringRef CWD, ArrayRef<std::string> CC1CommandLine, + IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS, + DiagnosticConsumer &DiagConsumer, DependencyActionController &Controller, + ArrayRef<StringRef> ModuleNames) { + std::optional<CompilerInstanceWithContext> CIWC = + CompilerInstanceWithContext::initializeFromCC1Commandline( + *this, CWD, CC1CommandLine, DiagConsumer, std::move(OverlayFS), + Controller); + if (!CIWC) + return std::nullopt; + return CIWC->computeModuleMapsForModules(ModuleNames); +} diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp index f7e94bcd44e44f..a65d87e20ad844 100644 --- a/clang/lib/Tooling/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanningTool.cpp @@ -372,11 +372,13 @@ getFirstCC1CommandLine(ArrayRef<std::string> CommandLine, return std::nullopt; } -bool DependencyScanningTool::getByNameDependencies( - StringRef CWD, ArrayRef<std::string> CommandLine, - DiagnosticConsumer &DiagConsumer, DependencyActionController &Controller, - llvm::function_ref<std::optional<std::string>()> getNextName, - DependencyConsumer &DepConsumer) { +/// Set up the overlay VFS and cc1 command line for a by-name query. +/// \returns std::nullopt if \p CommandLine is ill-formed. +static std::optional<std::pair<IntrusiveRefCntPtr<llvm::vfs::FileSystem>, + std::vector<std::string>>> +initByNameScanning(DependencyScanningWorker &Worker, StringRef CWD, + ArrayRef<std::string> CommandLine, + DiagnosticConsumer &DiagConsumer) { auto [OverlayFS, ModifiedCommandLine] = initVFSForByNameScanning(CommandLine); auto FS = Worker.makeEffectiveVFS(CWD, OverlayFS); std::vector<std::string> CC1CommandLine; @@ -391,11 +393,36 @@ bool DependencyScanningTool::getByNameDependencies( auto MaybeFirstCC1 = getFirstCC1CommandLine( ModifiedCommandLine, *DiagEngine, FS, Worker.getService().getLogger()); if (!MaybeFirstCC1) - return false; + return std::nullopt; CC1CommandLine.assign(MaybeFirstCC1->begin(), MaybeFirstCC1->end()); } + return std::make_pair(std::move(OverlayFS), std::move(CC1CommandLine)); +} +bool DependencyScanningTool::getByNameDependencies( + StringRef CWD, ArrayRef<std::string> CommandLine, + DiagnosticConsumer &DiagConsumer, DependencyActionController &Controller, + llvm::function_ref<std::optional<std::string>()> getNextName, + DependencyConsumer &DepConsumer) { + auto Init = initByNameScanning(Worker, CWD, CommandLine, DiagConsumer); + if (!Init) + return false; + auto &[OverlayFS, CC1CommandLine] = *Init; return Worker.computeDependenciesByName(CWD, CC1CommandLine, std::move(OverlayFS), DiagConsumer, Controller, getNextName, DepConsumer); } + +std::optional<llvm::StringMap<std::string>> +DependencyScanningTool::getModuleMapsForModules( + StringRef CWD, ArrayRef<std::string> CommandLine, + DiagnosticConsumer &DiagConsumer, DependencyActionController &Controller, + ArrayRef<StringRef> ModuleNames) { + auto Init = initByNameScanning(Worker, CWD, CommandLine, DiagConsumer); + if (!Init) + return std::nullopt; + auto &[OverlayFS, CC1CommandLine] = *Init; + return Worker.computeModuleMapsForModules(CWD, CC1CommandLine, + std::move(OverlayFS), DiagConsumer, + Controller, ModuleNames); +} diff --git a/clang/unittests/Tooling/DependencyScannerTest.cpp b/clang/unittests/Tooling/DependencyScannerTest.cpp index 0131d4c122a686..f9a60b9c1f6e98 100644 --- a/clang/unittests/Tooling/DependencyScannerTest.cpp +++ b/clang/unittests/Tooling/DependencyScannerTest.cpp @@ -404,3 +404,188 @@ TEST(DependencyScanner, InvalidatedPathAfterModuleBuilt) { llvm::sys::fs::remove_directories(Root); } + +/// Test fixture for module map resolution without module loading. +class GetModuleMapsForModulesTest : public ::testing::Test { +protected: + GetModuleMapsForModulesTest() + : VFS(llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>()), + Service(makeServiceOptions(VFS)), ScanTool(Service) { + VFS->setCurrentWorkingDirectory(CWD); + } + + void addFile(StringRef Path, StringRef Contents) { + VFS->addFile(Path, 0, llvm::MemoryBuffer::getMemBufferCopy(Contents)); + } + + /// Wrapper to get the defined modulemaps for /p ModuleNames, without loading. + std::optional<llvm::StringMap<std::string>> + getModuleMaps(ArrayRef<std::string> CommandLine, + ArrayRef<StringRef> ModuleNames, + DiagnosticConsumer &DiagConsumer) { + CallbackActionController Controller( + CallbackActionController::lookupUnreachableModuleOutput); + return ScanTool.getModuleMapsForModules(CWD, CommandLine, DiagConsumer, + Controller, ModuleNames); + } + + /// Get the modulemap that defines the module \p Name by doing a full module + /// load. If not found, return an empty string. + std::string scanModuleMapFile(ArrayRef<std::string> CommandLine, + StringRef Name, + DiagnosticConsumer &DiagConsumer) { + llvm::DenseSet<ModuleID> AlreadySeen; + FullDependencyConsumer Consumer(AlreadySeen); + auto LookupModuleOutput = [](const ModuleDeps &MD, ModuleOutputKind) { + return "/root/out/" + MD.ID.ModuleName + ".pcm"; + }; + CallbackActionController Controller(LookupModuleOutput); + bool Queried = false; + auto GetNextName = [&Queried, Name]() -> std::optional<std::string> { + if (Queried) + return std::nullopt; + Queried = true; + return Name.str(); + }; + if (!ScanTool.getByNameDependencies(CWD, CommandLine, DiagConsumer, + Controller, GetNextName, Consumer)) + return ""; + TranslationUnitDeps Deps = Consumer.takeTranslationUnitDeps(); + for (const ModuleDeps &MD : Deps.ModuleGraph) + if (MD.ID.ModuleName == Name) + return MD.ClangModuleMapFile; + return ""; + } + + static constexpr StringRef CWD = "/root"; + IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> VFS; + DependencyScanningService Service; + DependencyScanningTool ScanTool; + +private: + static DependencyScanningServiceOptions + makeServiceOptions(IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> VFS) { + DependencyScanningServiceOptions Opts; + Opts.MakeVFS = [VFS] { return VFS; }; + return Opts; + } +}; + +TEST_F(GetModuleMapsForModulesTest, Found) { + std::vector<std::string> CommandLine = { + "clang", "-target", "x86_64-apple-macosx10.7", + "-c", "test.m", "-fmodules", + "-I/root/a", "-I/root/b"}; + addFile("/root/test.m", "\n"); + addFile("/root/a/module.modulemap", "module A { header \"a.h\" }\n"); + addFile("/root/a/a.h", "\n"); + addFile("/root/b/module.modulemap", "module B { header \"b.h\" }\n"); + addFile("/root/b/b.h", "\n"); + + TextDiagnosticBuffer DiagConsumer; + std::optional<llvm::StringMap<std::string>> ModuleMaps = + getModuleMaps(CommandLine, {"A", "B"}, DiagConsumer); + ASSERT_TRUE(ModuleMaps); + using llvm::sys::path::convert_to_slash; + EXPECT_EQ(ModuleMaps->size(), 2u); + EXPECT_EQ(convert_to_slash(ModuleMaps->lookup("A")), + "/root/a/module.modulemap"); + EXPECT_EQ(convert_to_slash(ModuleMaps->lookup("B")), + "/root/b/module.modulemap"); +} + +// Verify a module provided by a prebuilt module file is omitted, even though +// it's resolvable through search paths. +TEST_F(GetModuleMapsForModulesTest, PrebuiltModule) { + // Set up a scenario where 'A' is a prebuilt module but its modulemap is + // resolved during the lookup for a diff erent module 'B'. + std::vector<std::string> CommandLine = { + "clang", "-target", "x86_64-apple-macosx10.7", "-c", + "test.m", "-fmodules", "-fmodule-file=A=/root/a.pcm", "-I/root/a"}; + addFile("/root/test.m", "\n"); + addFile("/root/a.pcm", ""); + addFile("/root/a/module.modulemap", "module A { header \"a.h\" }\n" + "module B { header \"b.h\" }\n"); + addFile("/root/a/a.h", "\n"); + addFile("/root/a/b.h", "\n"); + + TextDiagnosticBuffer DiagConsumer; + std::optional<llvm::StringMap<std::string>> ModuleMaps = + getModuleMaps(CommandLine, {"A", "B"}, DiagConsumer); + ASSERT_TRUE(ModuleMaps); + EXPECT_EQ(ModuleMaps->size(), 1u); + EXPECT_EQ(llvm::sys::path::convert_to_slash(ModuleMaps->lookup("B")), + "/root/a/module.modulemap"); + EXPECT_FALSE(ModuleMaps->contains("A")); +} + +// Verify a module that is not found is omitted, leaving an empty result. +TEST_F(GetModuleMapsForModulesTest, NotFound) { + std::vector<std::string> CommandLine = { + "clang", "-target", "x86_64-apple-macosx10.7", "-c", "test.m", + "-fmodules", "-I/root/a"}; + addFile("/root/test.m", "\n"); + addFile("/root/a/a.h", "\n"); + + TextDiagnosticBuffer DiagConsumer; + std::optional<llvm::StringMap<std::string>> ModuleMaps = + getModuleMaps(CommandLine, {"Missing"}, DiagConsumer); + ASSERT_TRUE(ModuleMaps); + EXPECT_TRUE(ModuleMaps->empty()); +} + +// Verify a command line that cannot be set up reports +// an error. +TEST_F(GetModuleMapsForModulesTest, IllFormedCommandLine) { + std::vector<std::string> CommandLine = {"clang", "-cc1", "-not-a-real-flag"}; + + TextDiagnosticBuffer DiagConsumer; + std::optional<llvm::StringMap<std::string>> ModuleMaps = + getModuleMaps(CommandLine, {"A"}, DiagConsumer); + EXPECT_FALSE(ModuleMaps); + ASSERT_NE(DiagConsumer.err_begin(), DiagConsumer.err_end()); + EXPECT_EQ(DiagConsumer.err_begin()->second, + "unknown argument: '-not-a-real-flag'"); +} + +// Verify a resolution error isn't diagnosed by the lookup, but is by a +// later scan. +TEST_F(GetModuleMapsForModulesTest, SkipsRedefinition) { + // Set up module redefinition error scenario. + // Both search paths define a module named 'A', and the compiler + // instance encounters both. + std::vector<std::string> CommandLine = { + "clang", "-target", "x86_64-apple-macosx10.7", + "-c", "test.m", "-fmodules", + "-I/root/a", "-I/root/b"}; + addFile("/root/test.m", "\n"); + addFile("/root/a/module.modulemap", "module A { header \"a.h\" }\n"); + addFile("/root/a/a.h", "\n"); + addFile("/root/b/module.modulemap", "module A { header \"a.h\" }\n" + "module B { header \"b.h\" }\n"); + addFile("/root/b/a.h", "\n"); + addFile("/root/b/b.h", "\n"); + + auto getErrors = [](TextDiagnosticBuffer &DiagConsumer) { + std::vector<std::string> Errors; + for (auto It = DiagConsumer.err_begin(); It != DiagConsumer.err_end(); ++It) + Errors.push_back(It->second); + return Errors; + }; + + TextDiagnosticBuffer LookupDiags; + std::optional<llvm::StringMap<std::string>> ModuleMaps = + getModuleMaps(CommandLine, {"A", "B"}, LookupDiags); + ASSERT_TRUE(ModuleMaps); + using llvm::sys::path::convert_to_slash; + EXPECT_EQ(convert_to_slash(ModuleMaps->lookup("A")), + "/root/a/module.modulemap"); + EXPECT_EQ(convert_to_slash(ModuleMaps->lookup("B")), + "/root/b/module.modulemap"); + EXPECT_EQ(getErrors(LookupDiags), std::vector<std::string>{}); + + TextDiagnosticBuffer ScanDiags; + EXPECT_EQ(scanModuleMapFile(CommandLine, "B", ScanDiags), ""); + EXPECT_TRUE( + llvm::is_contained(getErrors(ScanDiags), "redefinition of module 'A'")); +} _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
