https://github.com/naveen-seth updated https://github.com/llvm/llvm-project/pull/172347
>From c7a6ab6f0a1b4693bf2be35c2a147a8d3fb4443e Mon Sep 17 00:00:00 2001 From: Naveen Seth Hanig <[email protected]> Date: Mon, 15 Dec 2025 19:23:52 +0100 Subject: [PATCH 01/10] [clang][DependencyScanning] Remove dependency on clangDriver from clangDependencyScanning This is the final patch in a series that removes the dependency of clangDependencyScanning on clangDriver, splitting the work from #169964 into smaller changes (see comment linked below). This patch updates the remaining parts of the scanning interface in DependencyScanningWorker to only operate on -cc1 / driver job command lines. This follows #171238, which applied this change to the by-name scanning API. This is part of a broader effort to support driver-managed builds for compilations using C++ named modules and/or Clang modules. It is required for linking the dependency scanning tooling against the driver without introducing cyclic dependencies, which would otherwise cause build failures when dynamic linking is enabled. --- .../DependencyScannerImpl.h | 18 -- .../DependencyScanningWorker.h | 58 ++++-- clang/lib/DependencyScanning/CMakeLists.txt | 1 - .../DependencyScannerImpl.cpp | 91 --------- .../DependencyScanningWorker.cpp | 181 +++++++++--------- clang/lib/Tooling/DependencyScanningTool.cpp | 153 ++++++++++++--- .../DependencyScanningWorkerTest.cpp | 12 +- 7 files changed, 271 insertions(+), 243 deletions(-) diff --git a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h index f43c7f70183fd..231873375a264 100644 --- a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h +++ b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h @@ -89,28 +89,10 @@ struct TextDiagnosticsPrinterWithOutput { DiagPrinter(DiagnosticsOS, *DiagOpts) {} }; -std::pair<std::unique_ptr<driver::Driver>, std::unique_ptr<driver::Compilation>> -buildCompilation(ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags, - IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, - llvm::BumpPtrAllocator &Alloc); - std::unique_ptr<CompilerInvocation> createCompilerInvocation(ArrayRef<std::string> CommandLine, DiagnosticsEngine &Diags); -std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>, - std::vector<std::string>> -initVFSForTUBufferScanning(IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS, - ArrayRef<std::string> CommandLine, - StringRef WorkingDirectory, - llvm::MemoryBufferRef TUBuffer); - -std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>, - std::vector<std::string>> -initVFSForByNameScanning(IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS, - ArrayRef<std::string> CommandLine, - StringRef WorkingDirectory, StringRef ModuleName); - bool initializeScanCompilerInstance( CompilerInstance &ScanInstance, IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, diff --git a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h index 489fba4ed3f6b..35f289cf12eb0 100644 --- a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h @@ -16,7 +16,7 @@ #include "clang/DependencyScanning/DependencyScanningService.h" #include "clang/DependencyScanning/ModuleDepCollector.h" #include "clang/Frontend/PCHContainerOperations.h" -#include "llvm/Support/Error.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBufferRef.h" #include "llvm/Support/VirtualFileSystem.h" @@ -93,11 +93,13 @@ class DependencyScanningWorker { ~DependencyScanningWorker(); - /// Run the dependency scanning tool for a given clang driver command-line, - /// and report the discovered dependencies to the provided consumer. If - /// TUBuffer is not nullopt, it is used as TU input for the dependency - /// scanning. Otherwise, the input should be included as part of the - /// command-line. + /// Run the dependency scanning tool for the given driver job command-line, + /// and report the discovered dependencies to the provided consumer. + /// + /// OverlayFS should be based on the Worker's dependency scanning file-system + /// and can be used to provide any input specified on the command-line as + /// in-memory file. If no overlay file-system is provided, the Worker's + /// dependency scanning file-system is used instead. /// /// \returns false if clang errors occurred (with diagnostics reported to /// \c DiagConsumer), true otherwise. @@ -105,17 +107,25 @@ class DependencyScanningWorker { StringRef WorkingDirectory, ArrayRef<std::string> CommandLine, DependencyConsumer &DepConsumer, DependencyActionController &Controller, DiagnosticConsumer &DiagConsumer, - std::optional<llvm::MemoryBufferRef> TUBuffer = std::nullopt); + llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS = + nullptr); - /// Run the dependency scanning tool for a given clang driver command-line - /// for a specific translation unit via file system or memory buffer. + /// Run the dependency scanning tool for all given driver job command-lines, + /// and report the discovered dependencies to the provided consumer. /// - /// \returns A \c StringError with the diagnostic output if clang errors - /// occurred, success otherwise. - llvm::Error computeDependencies( - StringRef WorkingDirectory, ArrayRef<std::string> CommandLine, - DependencyConsumer &Consumer, DependencyActionController &Controller, - std::optional<llvm::MemoryBufferRef> TUBuffer = std::nullopt); + /// OverlayFS should be based on the Worker's dependency scanning file-system + /// and can be used to provide any input specified on the command-line as + /// in-memory file. If no overlay file-system is provided, the Worker's + /// dependency scanning file-system is used instead. + /// + /// \returns false if clang errors occurred (with diagnostics reported to + /// \c Diags), true otherwise. + bool computeDependencies( + StringRef WorkingDirectory, ArrayRef<ArrayRef<std::string>> CommandLines, + DependencyConsumer &DepConsumer, DependencyActionController &Controller, + DiagnosticsEngine &Diags, + llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS = + nullptr); /// The three method below implements a new interface for by name /// dependency scanning. They together enable the dependency scanning worker @@ -176,12 +186,26 @@ class DependencyScanningWorker { /// Actually carries out the scan. If \c OverlayFS is provided, it must be /// based on top of DepFS. bool scanDependencies( - StringRef WorkingDirectory, ArrayRef<std::string> CommandLine, + StringRef WorkingDirectory, + ArrayRef<ArrayRef<std::string>> CC1CommandLines, DependencyConsumer &Consumer, DependencyActionController &Controller, - DiagnosticConsumer &DC, + DiagnosticsEngine &Diags, IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS = nullptr); }; +std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>, + std::vector<std::string>> +initVFSForTUBufferScanning(IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS, + ArrayRef<std::string> CommandLine, + StringRef WorkingDirectory, + llvm::MemoryBufferRef TUBuffer); + +std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>, + std::vector<std::string>> +initVFSForByNameScanning(IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS, + ArrayRef<std::string> CommandLine, + StringRef WorkingDirectory, StringRef ModuleName); + } // end namespace dependencies } // end namespace clang diff --git a/clang/lib/DependencyScanning/CMakeLists.txt b/clang/lib/DependencyScanning/CMakeLists.txt index 2976f7c236f2e..4c30c3ee57cfd 100644 --- a/clang/lib/DependencyScanning/CMakeLists.txt +++ b/clang/lib/DependencyScanning/CMakeLists.txt @@ -20,7 +20,6 @@ add_clang_library(clangDependencyScanning LINK_LIBS clangAST clangBasic - clangDriver clangFrontend clangLex clangSerialization diff --git a/clang/lib/DependencyScanning/DependencyScannerImpl.cpp b/clang/lib/DependencyScanning/DependencyScannerImpl.cpp index 201230d7d6a8e..7fb214eb2e630 100644 --- a/clang/lib/DependencyScanning/DependencyScannerImpl.cpp +++ b/clang/lib/DependencyScanning/DependencyScannerImpl.cpp @@ -379,42 +379,6 @@ DiagnosticsEngineWithDiagOpts::DiagnosticsEngineWithDiagOpts( /*ShouldOwnClient=*/false); } -std::pair<std::unique_ptr<driver::Driver>, std::unique_ptr<driver::Compilation>> -dependencies::buildCompilation(ArrayRef<std::string> ArgStrs, - DiagnosticsEngine &Diags, - IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, - llvm::BumpPtrAllocator &Alloc) { - SmallVector<const char *, 256> Argv; - Argv.reserve(ArgStrs.size()); - for (const std::string &Arg : ArgStrs) - Argv.push_back(Arg.c_str()); - - std::unique_ptr<driver::Driver> Driver = std::make_unique<driver::Driver>( - Argv[0], llvm::sys::getDefaultTargetTriple(), Diags, - "clang LLVM compiler", FS); - Driver->setTitle("clang_based_tool"); - - bool CLMode = driver::IsClangCL( - driver::getDriverMode(Argv[0], ArrayRef(Argv).slice(1))); - - if (llvm::Error E = - driver::expandResponseFiles(Argv, CLMode, Alloc, FS.get())) { - Diags.Report(diag::err_drv_expand_response_file) - << llvm::toString(std::move(E)); - return std::make_pair(nullptr, nullptr); - } - - std::unique_ptr<driver::Compilation> Compilation( - Driver->BuildCompilation(Argv)); - if (!Compilation) - return std::make_pair(nullptr, nullptr); - - if (Compilation->containsError()) - return std::make_pair(nullptr, nullptr); - - return std::make_pair(std::move(Driver), std::move(Compilation)); -} - std::unique_ptr<CompilerInvocation> dependencies::createCompilerInvocation(ArrayRef<std::string> CommandLine, DiagnosticsEngine &Diags) { @@ -430,61 +394,6 @@ dependencies::createCompilerInvocation(ArrayRef<std::string> CommandLine, return Invocation; } -std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>, - std::vector<std::string>> -dependencies::initVFSForTUBufferScanning( - IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS, - ArrayRef<std::string> CommandLine, StringRef WorkingDirectory, - llvm::MemoryBufferRef TUBuffer) { - // Reset what might have been modified in the previous worker invocation. - BaseFS->setCurrentWorkingDirectory(WorkingDirectory); - - auto OverlayFS = - llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS); - auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>(); - InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory); - auto InputPath = TUBuffer.getBufferIdentifier(); - InMemoryFS->addFile( - InputPath, 0, llvm::MemoryBuffer::getMemBufferCopy(TUBuffer.getBuffer())); - IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay = InMemoryFS; - - OverlayFS->pushOverlay(InMemoryOverlay); - std::vector<std::string> ModifiedCommandLine(CommandLine); - ModifiedCommandLine.emplace_back(InputPath); - - return std::make_pair(OverlayFS, ModifiedCommandLine); -} - -std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>, - std::vector<std::string>> -dependencies::initVFSForByNameScanning( - IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS, - ArrayRef<std::string> CommandLine, StringRef WorkingDirectory, - StringRef ModuleName) { - // Reset what might have been modified in the previous worker invocation. - BaseFS->setCurrentWorkingDirectory(WorkingDirectory); - - // If we're scanning based on a module name alone, we don't expect the client - // to provide us with an input file. However, the driver really wants to have - // one. Let's just make it up to make the driver happy. - auto OverlayFS = - llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS); - auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>(); - InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory); - SmallString<128> FakeInputPath; - // TODO: We should retry the creation if the path already exists. - llvm::sys::fs::createUniquePath(ModuleName + "-%%%%%%%%.input", FakeInputPath, - /*MakeAbsolute=*/false); - InMemoryFS->addFile(FakeInputPath, 0, llvm::MemoryBuffer::getMemBuffer("")); - IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay = InMemoryFS; - OverlayFS->pushOverlay(InMemoryOverlay); - - std::vector<std::string> ModifiedCommandLine(CommandLine); - ModifiedCommandLine.emplace_back(FakeInputPath); - - return std::make_pair(OverlayFS, ModifiedCommandLine); -} - bool dependencies::initializeScanCompilerInstance( CompilerInstance &ScanInstance, IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, diff --git a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp index ef16b14e7cc6e..1aa77c69a9bb5 100644 --- a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp @@ -13,6 +13,7 @@ #include "clang/Driver/Driver.h" #include "clang/Driver/Tool.h" #include "clang/Serialization/ObjectFilePCHContainerReader.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/Support/VirtualFileSystem.h" using namespace clang; @@ -40,39 +41,6 @@ DependencyScanningWorker::DependencyScanningWorker( DependencyScanningWorker::~DependencyScanningWorker() = default; DependencyActionController::~DependencyActionController() = default; -llvm::Error DependencyScanningWorker::computeDependencies( - StringRef WorkingDirectory, ArrayRef<std::string> CommandLine, - DependencyConsumer &Consumer, DependencyActionController &Controller, - std::optional<llvm::MemoryBufferRef> TUBuffer) { - // Capture the emitted diagnostics and report them to the client - // in the case of a failure. - TextDiagnosticsPrinterWithOutput DiagPrinterWithOS(CommandLine); - - if (computeDependencies(WorkingDirectory, CommandLine, Consumer, Controller, - DiagPrinterWithOS.DiagPrinter, TUBuffer)) - return llvm::Error::success(); - return llvm::make_error<llvm::StringError>( - DiagPrinterWithOS.DiagnosticsOS.str(), llvm::inconvertibleErrorCode()); -} - -static bool forEachDriverJob( - ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags, - IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, - llvm::function_ref<bool(const driver::Command &Cmd)> Callback) { - // Compilation holds a non-owning a reference to the Driver, hence we need to - // keep the Driver alive when we use Compilation. Arguments to commands may be - // owned by Alloc when expanded from response files. - llvm::BumpPtrAllocator Alloc; - auto [Driver, Compilation] = buildCompilation(ArgStrs, Diags, FS, Alloc); - if (!Compilation) - return false; - for (const driver::Command &Job : Compilation->getJobs()) { - if (!Callback(Job)) - return false; - } - return true; -} - static bool createAndRunToolInvocation( ArrayRef<std::string> CommandLine, DependencyScanningAction &Action, IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, @@ -88,11 +56,11 @@ static bool createAndRunToolInvocation( } bool DependencyScanningWorker::scanDependencies( - StringRef WorkingDirectory, ArrayRef<std::string> CommandLine, + StringRef WorkingDirectory, ArrayRef<ArrayRef<std::string>> CC1CommandLines, DependencyConsumer &Consumer, DependencyActionController &Controller, - DiagnosticConsumer &DC, - IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) { - IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = DepFS; + DiagnosticsEngine &Diags, + llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) { + IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = nullptr; if (OverlayFS) { #ifndef NDEBUG bool SawDepFS = false; @@ -101,71 +69,55 @@ bool DependencyScanningWorker::scanDependencies( assert(SawDepFS && "OverlayFS not based on DepFS"); #endif FS = std::move(OverlayFS); + } else { + FS = DepFS; + FS->setCurrentWorkingDirectory(WorkingDirectory); } - DiagnosticsEngineWithDiagOpts DiagEngineWithCmdAndOpts(CommandLine, FS, DC); DependencyScanningAction Action(Service, WorkingDirectory, Consumer, Controller, DepFS); - bool Success = false; - if (CommandLine[1] == "-cc1") { - Success = - createAndRunToolInvocation(CommandLine, Action, FS, PCHContainerOps, - *DiagEngineWithCmdAndOpts.DiagEngine); - } else { - Success = forEachDriverJob( - CommandLine, *DiagEngineWithCmdAndOpts.DiagEngine, FS, - [&](const driver::Command &Cmd) { - if (StringRef(Cmd.getCreator().getName()) != "clang") { - // Non-clang command. Just pass through to the dependency - // consumer. - Consumer.handleBuildCommand( - {Cmd.getExecutable(), - {Cmd.getArguments().begin(), Cmd.getArguments().end()}}); - return true; - } - - // Insert -cc1 command line options into Argv - std::vector<std::string> Argv; - Argv.push_back(Cmd.getExecutable()); - llvm::append_range(Argv, Cmd.getArguments()); - - // Create an invocation that uses the underlying file - // system to ensure that any file system requests that - // are made by the driver do not go through the - // dependency scanning filesystem. - return createAndRunToolInvocation( - std::move(Argv), Action, FS, PCHContainerOps, - *DiagEngineWithCmdAndOpts.DiagEngine); - }); - } - - if (Success && !Action.hasScanned()) - DiagEngineWithCmdAndOpts.DiagEngine->Report( - diag::err_fe_expected_compiler_job) - << llvm::join(CommandLine, " "); + const bool Success = llvm::all_of(CC1CommandLines, [&](const auto &Cmd) { + if (StringRef(Cmd[1]) != "-cc1") { + // Non-clang command. Just pass through to the dependency consumer. + Consumer.handleBuildCommand({Cmd.front(), {Cmd.begin() + 1, Cmd.end()}}); + return true; + } + // Create an invocation that uses the underlying file system to ensure that + // any file system requests that are made by the driver do not go through + // the dependency scanning filesystem. + return createAndRunToolInvocation(Cmd, Action, FS, PCHContainerOps, Diags); + }); // Ensure finish() is called even if we never reached ExecuteAction(). if (!Action.hasDiagConsumerFinished()) - DC.finish(); + Diags.getClient()->finish(); return Success && Action.hasScanned(); } bool DependencyScanningWorker::computeDependencies( StringRef WorkingDirectory, ArrayRef<std::string> CommandLine, - DependencyConsumer &Consumer, DependencyActionController &Controller, - DiagnosticConsumer &DC, std::optional<llvm::MemoryBufferRef> TUBuffer) { - if (TUBuffer) { - auto [FinalFS, FinalCommandLine] = initVFSForTUBufferScanning( - DepFS, CommandLine, WorkingDirectory, *TUBuffer); - return scanDependencies(WorkingDirectory, FinalCommandLine, Consumer, - Controller, DC, FinalFS); - } + DependencyConsumer &DepConsumer, DependencyActionController &Controller, + DiagnosticConsumer &DiagConsumer, + llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS) { + DiagnosticsEngineWithDiagOpts DiagEngineWithDiagOpts = + OverlayFS + ? DiagnosticsEngineWithDiagOpts(CommandLine, OverlayFS, DiagConsumer) + : DiagnosticsEngineWithDiagOpts(CommandLine, DepFS, DiagConsumer); + auto &Diags = *DiagEngineWithDiagOpts.DiagEngine; + return computeDependencies(WorkingDirectory, + ArrayRef<ArrayRef<std::string>>(CommandLine), + DepConsumer, Controller, Diags, OverlayFS); +} - DepFS->setCurrentWorkingDirectory(WorkingDirectory); - return scanDependencies(WorkingDirectory, CommandLine, Consumer, Controller, - DC); +bool DependencyScanningWorker::computeDependencies( + StringRef WorkingDirectory, ArrayRef<ArrayRef<std::string>> CommandLines, + DependencyConsumer &DepConsumer, DependencyActionController &Controller, + DiagnosticsEngine &Diags, + llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS) { + return scanDependencies(WorkingDirectory, CommandLines, DepConsumer, + Controller, Diags, OverlayFS); } bool DependencyScanningWorker::initializeCompilerInstanceWithContext( @@ -203,3 +155,58 @@ bool DependencyScanningWorker::computeDependenciesByNameWithContext( bool DependencyScanningWorker::finalizeCompilerInstanceWithContext() { return CIWithContext->finalize(); } + +std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>, + std::vector<std::string>> +dependencies::initVFSForTUBufferScanning( + IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS, + ArrayRef<std::string> CommandLine, StringRef WorkingDirectory, + llvm::MemoryBufferRef TUBuffer) { + // Reset what might have been modified in the previous worker invocation. + BaseFS->setCurrentWorkingDirectory(WorkingDirectory); + + auto OverlayFS = + llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS); + auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>(); + InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory); + auto InputPath = TUBuffer.getBufferIdentifier(); + InMemoryFS->addFile( + InputPath, 0, llvm::MemoryBuffer::getMemBufferCopy(TUBuffer.getBuffer())); + IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay = InMemoryFS; + + OverlayFS->pushOverlay(InMemoryOverlay); + std::vector<std::string> ModifiedCommandLine(CommandLine); + ModifiedCommandLine.emplace_back(InputPath); + + return std::make_pair(OverlayFS, ModifiedCommandLine); +} + +std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>, + std::vector<std::string>> +dependencies::initVFSForByNameScanning( + IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS, + ArrayRef<std::string> CommandLine, StringRef WorkingDirectory, + StringRef ModuleName) { + // Reset what might have been modified in the previous worker invocation. + BaseFS->setCurrentWorkingDirectory(WorkingDirectory); + + // If we're scanning based on a module name alone, we don't expect the client + // to provide us with an input file. However, the driver really wants to have + // one. Let's just make it up to make the driver happy. + auto OverlayFS = + llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS); + auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>(); + InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory); + SmallString<128> FakeInputPath; + // TODO: We should retry the creation if the path already exists. + llvm::sys::fs::createUniquePath(ModuleName + "-%%%%%%%%.input", FakeInputPath, + /*MakeAbsolute=*/false); + InMemoryFS->addFile(FakeInputPath, 0, llvm::MemoryBuffer::getMemBuffer("")); + IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay = InMemoryFS; + OverlayFS->pushOverlay(InMemoryOverlay); + + std::vector<std::string> ModifiedCommandLine(CommandLine); + ModifiedCommandLine.emplace_back(FakeInputPath); + + return std::make_pair(OverlayFS, ModifiedCommandLine); +} diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp index 74cc6af3551f8..1ca81fc9a0d83 100644 --- a/clang/lib/Tooling/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanningTool.cpp @@ -7,10 +7,15 @@ //===----------------------------------------------------------------------===// #include "clang/Tooling/DependencyScanningTool.h" +#include "clang/Basic/Diagnostic.h" #include "clang/Basic/DiagnosticFrontend.h" #include "clang/DependencyScanning/DependencyScannerImpl.h" #include "clang/Driver/Tool.h" #include "clang/Frontend/Utils.h" +#include "llvm/ADT/SmallVectorExtras.h" +#include "llvm/ADT/iterator.h" +#include "llvm/Support/VirtualFileSystem.h" +#include "llvm/TargetParser/Host.h" #include <optional> using namespace clang; @@ -74,13 +79,131 @@ class MakeDependencyPrinterConsumer : public DependencyConsumer { }; } // anonymous namespace +static std::pair<std::unique_ptr<driver::Driver>, + std::unique_ptr<driver::Compilation>> +buildCompilation(ArrayRef<std::string> CommandLine, DiagnosticsEngine &Diags, + IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, + llvm::BumpPtrAllocator &Alloc) { + auto Argv = llvm::map_to_vector<256>( + CommandLine, [](const auto &Arg) { return Arg.c_str(); }); + + std::unique_ptr<driver::Driver> Driver = std::make_unique<driver::Driver>( + Argv[0], llvm::sys::getDefaultTargetTriple(), Diags, + "clang LLVM compiler", FS); + Driver->setTitle("clang_based_tool"); + + bool CLMode = driver::IsClangCL( + driver::getDriverMode(Argv[0], ArrayRef(Argv).slice(1))); + + if (llvm::Error E = + driver::expandResponseFiles(Argv, CLMode, Alloc, FS.get())) { + Diags.Report(diag::err_drv_expand_response_file) + << llvm::toString(std::move(E)); + return std::make_pair(nullptr, nullptr); + } + + std::unique_ptr<driver::Compilation> Compilation( + Driver->BuildCompilation(Argv)); + if (!Compilation) + return std::make_pair(nullptr, nullptr); + + if (Compilation->containsError()) + return std::make_pair(nullptr, nullptr); + + if (Compilation->getJobs().empty()) { + Diags.Report(diag::err_fe_expected_compiler_job) + << llvm::join(CommandLine, " "); + return std::make_pair(nullptr, nullptr); + } + + return std::make_pair(std::move(Driver), std::move(Compilation)); +} + +/// Constructs the full -cc1 command line, including executable, for the given +/// driver \c Cmd. +static std::vector<std::string> +buildCC1CommandLine(const driver::Command &Cmd) { + const auto &Args = Cmd.getArguments(); + std::vector<std::string> Out; + Out.reserve(Args.size() + 1); + Out.emplace_back(Cmd.getExecutable()); + llvm::append_range(Out, Args); + return Out; +} + +static bool computeDependenciesForDriverCommandLine( + DependencyScanningWorker &Worker, StringRef WorkingDirectory, + ArrayRef<std::string> CommandLine, DependencyConsumer &Consumer, + DependencyActionController &Controller, DiagnosticsEngine &Diags, + IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS) { + Worker.getVFS().setCurrentWorkingDirectory(WorkingDirectory); + + // Compilation holds a non-owning a reference to the Driver, hence we need to + // keep the Driver alive when we use Compilation. Arguments to commands may be + // owned by Alloc when expanded from response files. + llvm::BumpPtrAllocator Alloc; + const auto [Driver, Compilation] = + buildCompilation(CommandLine, Diags, &Worker.getVFS(), Alloc); + if (!Compilation) + return false; + + const auto CC1CommandLines = + llvm::map_to_vector(Compilation->getJobs(), buildCC1CommandLine); + const auto CC1CommandLinesView = llvm::map_to_vector( + CC1CommandLines, [](const auto &CC1Cmds) { return ArrayRef(CC1Cmds); }); + + return Worker.computeDependencies(WorkingDirectory, CC1CommandLinesView, + Consumer, Controller, Diags, OverlayFS); +} + +static llvm::Error makeErrorFromDiagnosticsOS( + TextDiagnosticsPrinterWithOutput &DiagPrinterWithOS) { + return llvm::make_error<llvm::StringError>( + DiagPrinterWithOS.DiagnosticsOS.str(), llvm::inconvertibleErrorCode()); +} + +static llvm::Error computeDependenciesOrError( + DependencyScanningWorker &Worker, StringRef WorkingDirectory, + ArrayRef<std::string> CommandLine, DependencyConsumer &Consumer, + DependencyActionController &Controller, + std::optional<llvm::MemoryBufferRef> TUBuffer = std::nullopt) { + // If we are scanning from a TUBuffer, create an overlay filesystem with the + // input as an in-memory file and add it to the command line. + IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS = nullptr; + std::vector<std::string> CommandLineWithTUBufferInput; + if (TUBuffer) { + std::tie(OverlayFS, CommandLineWithTUBufferInput) = + initVFSForTUBufferScanning(&Worker.getVFS(), CommandLine, + WorkingDirectory, *TUBuffer); + CommandLine = CommandLineWithTUBufferInput; + } + + TextDiagnosticsPrinterWithOutput DiagPrinterWithOS(CommandLine); + DiagnosticsEngineWithDiagOpts DiagEngineWithDiagOpts( + CommandLine, &Worker.getVFS(), DiagPrinterWithOS.DiagPrinter); + auto &Diags = *DiagEngineWithDiagOpts.DiagEngine; + + const auto IsCC1Input = (CommandLine.size() >= 2 && CommandLine[1] == "-cc1"); + const auto Success = + IsCC1Input + ? Worker.computeDependencies(WorkingDirectory, CommandLine, Consumer, + Controller, Diags, OverlayFS) + : computeDependenciesForDriverCommandLine( + Worker, WorkingDirectory, CommandLine, Consumer, Controller, + Diags, OverlayFS); + + if (!Success) + return makeErrorFromDiagnosticsOS(DiagPrinterWithOS); + return llvm::Error::success(); +} + llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(ArrayRef<std::string> CommandLine, StringRef CWD) { MakeDependencyPrinterConsumer Consumer; CallbackActionController Controller(nullptr); - auto Result = - Worker.computeDependencies(CWD, CommandLine, Consumer, Controller); + auto Result = computeDependenciesOrError(Worker, CWD, CommandLine, Consumer, + Controller); if (Result) return std::move(Result); std::string Output; @@ -132,8 +255,8 @@ llvm::Expected<P1689Rule> DependencyScanningTool::getP1689ModuleDependencyFile( P1689Rule Rule; P1689ModuleDependencyPrinterConsumer Consumer(Rule, Command); P1689ActionController Controller; - auto Result = Worker.computeDependencies(CWD, Command.CommandLine, Consumer, - Controller); + auto Result = computeDependenciesOrError(Worker, CWD, Command.CommandLine, + Consumer, Controller); if (Result) return std::move(Result); @@ -151,8 +274,8 @@ DependencyScanningTool::getTranslationUnitDependencies( std::optional<llvm::MemoryBufferRef> TUBuffer) { FullDependencyConsumer Consumer(AlreadySeen); CallbackActionController Controller(LookupModuleOutput); - llvm::Error Result = Worker.computeDependencies(CWD, CommandLine, Consumer, - Controller, TUBuffer); + llvm::Error Result = computeDependenciesOrError( + Worker, CWD, CommandLine, Consumer, Controller, TUBuffer); if (Result) return std::move(Result); @@ -177,18 +300,6 @@ DependencyScanningTool::getModuleDependencies( return Result; } -/// Constructs the full -cc1 command line, including executable, for the given -/// driver \c Cmd. -static std::vector<std::string> -buildCC1CommandLine(const driver::Command &Cmd) { - const auto &Args = Cmd.getArguments(); - std::vector<std::string> Out; - Out.reserve(Args.size() + 1); - Out.emplace_back(Cmd.getExecutable()); - llvm::append_range(Out, Args); - return Out; -} - static std::optional<std::vector<std::string>> getFirstCC1CommandLine( ArrayRef<std::string> CommandLine, DiagnosticsEngine &Diags, llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> ScanFS) { @@ -211,12 +322,6 @@ static std::optional<std::vector<std::string>> getFirstCC1CommandLine( return std::nullopt; } -static llvm::Error makeErrorFromDiagnosticsOS( - TextDiagnosticsPrinterWithOutput &DiagPrinterWithOS) { - return llvm::make_error<llvm::StringError>( - DiagPrinterWithOS.DiagnosticsOS.str(), llvm::inconvertibleErrorCode()); -} - llvm::Error DependencyScanningTool::initializeCompilerInstanceWithContextOrError( StringRef CWD, ArrayRef<std::string> CommandLine) { diff --git a/clang/unittests/DependencyScanning/DependencyScanningWorkerTest.cpp b/clang/unittests/DependencyScanning/DependencyScanningWorkerTest.cpp index e6a5684b10cc9..872c7effde3d3 100644 --- a/clang/unittests/DependencyScanning/DependencyScanningWorkerTest.cpp +++ b/clang/unittests/DependencyScanning/DependencyScanningWorkerTest.cpp @@ -47,9 +47,10 @@ TEST(DependencyScanner, ScanDepsWithDiagConsumer) { { // Check that a successful scan calls DiagConsumer.finish(). std::vector<std::string> Args = {"clang", - "-target", + "-cc1", + "-triple", "x86_64-apple-macosx10.7", - "-c", + "-emit-obj", "test.cpp", "-o" "test.cpp.o"}; @@ -65,7 +66,7 @@ TEST(DependencyScanner, ScanDepsWithDiagConsumer) { { // Check that an invalid command-line, which never enters the scanning // action calls DiagConsumer.finish(). - std::vector<std::string> Args = {"clang", "-invalid-arg"}; + std::vector<std::string> Args = {"clang", "-cc1", "-invalid-arg"}; EnsureFinishedConsumer DiagConsumer; bool Success = Worker.computeDependencies(CWD, Args, DC, AC, DiagConsumer); @@ -78,9 +79,10 @@ TEST(DependencyScanner, ScanDepsWithDiagConsumer) { // Check that a valid command line that produces no scanning jobs calls // DiagConsumer.finish(). std::vector<std::string> Args = {"clang", - "-target", + "-cc1", + "-triple", "x86_64-apple-macosx10.7", - "-c", + "-emit-obj", "-x", "assembler", "test.s", >From d5dae83a72114fc04df4d4a3d9a43bf6eb7a036c Mon Sep 17 00:00:00 2001 From: Naveen Seth Hanig <[email protected]> Date: Mon, 15 Dec 2025 21:17:35 +0100 Subject: [PATCH 02/10] Apply suggestions from code review Co-authored-by: Jan Svoboda <[email protected]> --- .../clang/DependencyScanning/DependencyScanningWorker.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h index 35f289cf12eb0..f46e2b2236604 100644 --- a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h @@ -93,7 +93,7 @@ class DependencyScanningWorker { ~DependencyScanningWorker(); - /// Run the dependency scanning tool for the given driver job command-line, + /// Run the dependency scanning worker for the given frontend command-line, /// and report the discovered dependencies to the provided consumer. /// /// OverlayFS should be based on the Worker's dependency scanning file-system @@ -110,7 +110,7 @@ class DependencyScanningWorker { llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS = nullptr); - /// Run the dependency scanning tool for all given driver job command-lines, + /// Run the dependency scanning tool for all given frontend command-lines, /// and report the discovered dependencies to the provided consumer. /// /// OverlayFS should be based on the Worker's dependency scanning file-system >From af9ae17496f8f9aa0a7d2c5cecf4518969da1fbe Mon Sep 17 00:00:00 2001 From: Naveen Seth Hanig <[email protected]> Date: Tue, 16 Dec 2025 19:18:08 +0100 Subject: [PATCH 03/10] Revert buildCompilation changes --- clang/lib/Tooling/DependencyScanningTool.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp index 9adf2422ae840..37bc72d890682 100644 --- a/clang/lib/Tooling/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanningTool.cpp @@ -81,11 +81,13 @@ class MakeDependencyPrinterConsumer : public DependencyConsumer { static std::pair<std::unique_ptr<driver::Driver>, std::unique_ptr<driver::Compilation>> -buildCompilation(ArrayRef<std::string> CommandLine, DiagnosticsEngine &Diags, +buildCompilation(ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags, IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, llvm::BumpPtrAllocator &Alloc) { - auto Argv = llvm::map_to_vector<256>( - CommandLine, [](const auto &Arg) { return Arg.c_str(); }); + SmallVector<const char *, 256> Argv; + Argv.reserve(ArgStrs.size()); + for (const std::string &Arg : ArgStrs) + Argv.push_back(Arg.c_str()); std::unique_ptr<driver::Driver> Driver = std::make_unique<driver::Driver>( Argv[0], llvm::sys::getDefaultTargetTriple(), Diags, @@ -112,7 +114,7 @@ buildCompilation(ArrayRef<std::string> CommandLine, DiagnosticsEngine &Diags, if (Compilation->getJobs().empty()) { Diags.Report(diag::err_fe_expected_compiler_job) - << llvm::join(CommandLine, " "); + << llvm::join(ArgStrs, " "); return std::make_pair(nullptr, nullptr); } >From 401ee99764897b727b014f0b6c2c44e3e74cbd02 Mon Sep 17 00:00:00 2001 From: Naveen Seth Hanig <[email protected]> Date: Tue, 16 Dec 2025 19:39:30 +0100 Subject: [PATCH 04/10] Move away from map_to_vector, prefer SmallVector Now that we are using ArrayRef consistently in the computeDependencies function signatures, we can also use SmallVector over std::vector as per https://llvm.org/docs/ProgrammersManual.html#llvm-adt-smallvector-h --- clang/lib/Tooling/DependencyScanningTool.cpp | 25 ++++++++++---------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp index 37bc72d890682..1aaa46c5266b1 100644 --- a/clang/lib/Tooling/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanningTool.cpp @@ -121,12 +121,12 @@ buildCompilation(ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags, return std::make_pair(std::move(Driver), std::move(Compilation)); } -/// Constructs the full -cc1 command line, including executable, for the given -/// driver \c Cmd. -static std::vector<std::string> -buildCC1CommandLine(const driver::Command &Cmd) { +/// Constructs the full frontend command line, including executable, for the +/// given driver \c Cmd. +static SmallVector<std::string, 0> +buildFrontendCommandLine(const driver::Command &Cmd) { const auto &Args = Cmd.getArguments(); - std::vector<std::string> Out; + SmallVector<std::string, 0> Out; Out.reserve(Args.size() + 1); Out.emplace_back(Cmd.getExecutable()); llvm::append_range(Out, Args); @@ -149,12 +149,13 @@ static bool computeDependenciesForDriverCommandLine( if (!Compilation) return false; - const auto CC1CommandLines = - llvm::map_to_vector(Compilation->getJobs(), buildCC1CommandLine); - const auto CC1CommandLinesView = llvm::map_to_vector( - CC1CommandLines, [](const auto &CC1Cmds) { return ArrayRef(CC1Cmds); }); + SmallVector<SmallVector<std::string, 0>> FrontendCommandLines; + for (const auto &Cmd : Compilation->getJobs()) + FrontendCommandLines.push_back(buildFrontendCommandLine(Cmd)); + SmallVector<ArrayRef<std::string>> FrontendCommandLinesView( + FrontendCommandLines.begin(), FrontendCommandLines.end()); - return Worker.computeDependencies(WorkingDirectory, CC1CommandLinesView, + return Worker.computeDependencies(WorkingDirectory, FrontendCommandLinesView, Consumer, Controller, Diags, OverlayFS); } @@ -296,7 +297,7 @@ DependencyScanningTool::getModuleDependencies( return Result; } -static std::optional<std::vector<std::string>> getFirstCC1CommandLine( +static std::optional<SmallVector<std::string, 0>> getFirstCC1CommandLine( ArrayRef<std::string> CommandLine, DiagnosticsEngine &Diags, llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> ScanFS) { // Compilation holds a non-owning a reference to the Driver, hence we need to @@ -314,7 +315,7 @@ static std::optional<std::vector<std::string>> getFirstCC1CommandLine( const auto &Jobs = Compilation->getJobs(); if (const auto It = llvm::find_if(Jobs, IsClangCmd); It != Jobs.end()) - return buildCC1CommandLine(*It); + return buildFrontendCommandLine(*It); return std::nullopt; } >From d2e53d67099e94e272d16ec1ee58e76050323c2f Mon Sep 17 00:00:00 2001 From: Naveen Seth Hanig <[email protected]> Date: Tue, 16 Dec 2025 19:49:40 +0100 Subject: [PATCH 05/10] Inline scanDependencies --- .../DependencyScanningWorker.h | 9 ---- .../DependencyScanningWorker.cpp | 54 ++++++++----------- 2 files changed, 23 insertions(+), 40 deletions(-) diff --git a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h index f46e2b2236604..25f03501119a1 100644 --- a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h @@ -182,15 +182,6 @@ class DependencyScanningWorker { friend CompilerInstanceWithContext; std::unique_ptr<CompilerInstanceWithContext> CIWithContext; - - /// Actually carries out the scan. If \c OverlayFS is provided, it must be - /// based on top of DepFS. - bool scanDependencies( - StringRef WorkingDirectory, - ArrayRef<ArrayRef<std::string>> CC1CommandLines, - DependencyConsumer &Consumer, DependencyActionController &Controller, - DiagnosticsEngine &Diags, - IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS = nullptr); }; std::pair<IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem>, diff --git a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp index 58cd28395bf81..089e792990859 100644 --- a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp @@ -55,11 +55,26 @@ static bool createAndRunToolInvocation( Diags.getClient()); } -bool DependencyScanningWorker::scanDependencies( - StringRef WorkingDirectory, ArrayRef<ArrayRef<std::string>> CC1CommandLines, - DependencyConsumer &Consumer, DependencyActionController &Controller, +bool DependencyScanningWorker::computeDependencies( + StringRef WorkingDirectory, ArrayRef<std::string> CommandLine, + DependencyConsumer &DepConsumer, DependencyActionController &Controller, + DiagnosticConsumer &DiagConsumer, + llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS) { + DiagnosticsEngineWithDiagOpts DiagEngineWithDiagOpts = + OverlayFS + ? DiagnosticsEngineWithDiagOpts(CommandLine, OverlayFS, DiagConsumer) + : DiagnosticsEngineWithDiagOpts(CommandLine, DepFS, DiagConsumer); + auto &Diags = *DiagEngineWithDiagOpts.DiagEngine; + return computeDependencies(WorkingDirectory, + ArrayRef<ArrayRef<std::string>>(CommandLine), + DepConsumer, Controller, Diags, OverlayFS); +} + +bool DependencyScanningWorker::computeDependencies( + StringRef WorkingDirectory, ArrayRef<ArrayRef<std::string>> CommandLines, + DependencyConsumer &DepConsumer, DependencyActionController &Controller, DiagnosticsEngine &Diags, - llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> OverlayFS) { + llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS) { IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = nullptr; if (OverlayFS) { #ifndef NDEBUG @@ -74,13 +89,14 @@ bool DependencyScanningWorker::scanDependencies( FS->setCurrentWorkingDirectory(WorkingDirectory); } - DependencyScanningAction Action(Service, WorkingDirectory, Consumer, + DependencyScanningAction Action(Service, WorkingDirectory, DepConsumer, Controller, DepFS); - const bool Success = llvm::all_of(CC1CommandLines, [&](const auto &Cmd) { + const bool Success = llvm::all_of(CommandLines, [&](const auto &Cmd) { if (StringRef(Cmd[1]) != "-cc1") { // Non-clang command. Just pass through to the dependency consumer. - Consumer.handleBuildCommand({Cmd.front(), {Cmd.begin() + 1, Cmd.end()}}); + DepConsumer.handleBuildCommand( + {Cmd.front(), {Cmd.begin() + 1, Cmd.end()}}); return true; } // Create an invocation that uses the underlying file system to ensure that @@ -96,30 +112,6 @@ bool DependencyScanningWorker::scanDependencies( return Success && Action.hasScanned(); } -bool DependencyScanningWorker::computeDependencies( - StringRef WorkingDirectory, ArrayRef<std::string> CommandLine, - DependencyConsumer &DepConsumer, DependencyActionController &Controller, - DiagnosticConsumer &DiagConsumer, - llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS) { - DiagnosticsEngineWithDiagOpts DiagEngineWithDiagOpts = - OverlayFS - ? DiagnosticsEngineWithDiagOpts(CommandLine, OverlayFS, DiagConsumer) - : DiagnosticsEngineWithDiagOpts(CommandLine, DepFS, DiagConsumer); - auto &Diags = *DiagEngineWithDiagOpts.DiagEngine; - return computeDependencies(WorkingDirectory, - ArrayRef<ArrayRef<std::string>>(CommandLine), - DepConsumer, Controller, Diags, OverlayFS); -} - -bool DependencyScanningWorker::computeDependencies( - StringRef WorkingDirectory, ArrayRef<ArrayRef<std::string>> CommandLines, - DependencyConsumer &DepConsumer, DependencyActionController &Controller, - DiagnosticsEngine &Diags, - llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS) { - return scanDependencies(WorkingDirectory, CommandLines, DepConsumer, - Controller, Diags, OverlayFS); -} - bool DependencyScanningWorker::initializeCompilerInstanceWithContext( StringRef CWD, ArrayRef<std::string> CommandLine, DiagnosticConsumer &DC) { auto [OverlayFS, ModifiedCommandLine] = >From bd8b27248565a1f57db471ddcd602e64f32c5882 Mon Sep 17 00:00:00 2001 From: Naveen Seth Hanig <[email protected]> Date: Tue, 16 Dec 2025 19:53:24 +0100 Subject: [PATCH 06/10] auto use --- clang/lib/DependencyScanning/DependencyScanningWorker.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp index 089e792990859..355217c61cbdc 100644 --- a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp @@ -60,7 +60,7 @@ bool DependencyScanningWorker::computeDependencies( DependencyConsumer &DepConsumer, DependencyActionController &Controller, DiagnosticConsumer &DiagConsumer, llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS) { - DiagnosticsEngineWithDiagOpts DiagEngineWithDiagOpts = + auto DiagEngineWithDiagOpts = OverlayFS ? DiagnosticsEngineWithDiagOpts(CommandLine, OverlayFS, DiagConsumer) : DiagnosticsEngineWithDiagOpts(CommandLine, DepFS, DiagConsumer); >From 0bface72b78ca9e8bfb16a4a74560957af575c99 Mon Sep 17 00:00:00 2001 From: Naveen Seth Hanig <[email protected]> Date: Tue, 16 Dec 2025 20:16:21 +0100 Subject: [PATCH 07/10] Align computeDependencies signatures, use new DiagnosticEngine for each frontend invocation. --- .../DependencyScanningWorker.h | 4 +-- .../DependencyScanningWorker.cpp | 16 ++++----- clang/lib/Tooling/DependencyScanningTool.cpp | 36 ++++++++++--------- 3 files changed, 30 insertions(+), 26 deletions(-) diff --git a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h index 25f03501119a1..75a84e378b5fa 100644 --- a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h @@ -119,11 +119,11 @@ class DependencyScanningWorker { /// dependency scanning file-system is used instead. /// /// \returns false if clang errors occurred (with diagnostics reported to - /// \c Diags), true otherwise. + /// \c DiagConsumer), true otherwise. bool computeDependencies( StringRef WorkingDirectory, ArrayRef<ArrayRef<std::string>> CommandLines, DependencyConsumer &DepConsumer, DependencyActionController &Controller, - DiagnosticsEngine &Diags, + DiagnosticConsumer &DiagConsumer, llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS = nullptr); diff --git a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp index 355217c61cbdc..266660cdfab28 100644 --- a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp @@ -60,20 +60,15 @@ bool DependencyScanningWorker::computeDependencies( DependencyConsumer &DepConsumer, DependencyActionController &Controller, DiagnosticConsumer &DiagConsumer, llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS) { - auto DiagEngineWithDiagOpts = - OverlayFS - ? DiagnosticsEngineWithDiagOpts(CommandLine, OverlayFS, DiagConsumer) - : DiagnosticsEngineWithDiagOpts(CommandLine, DepFS, DiagConsumer); - auto &Diags = *DiagEngineWithDiagOpts.DiagEngine; return computeDependencies(WorkingDirectory, ArrayRef<ArrayRef<std::string>>(CommandLine), - DepConsumer, Controller, Diags, OverlayFS); + DepConsumer, Controller, DiagConsumer, OverlayFS); } bool DependencyScanningWorker::computeDependencies( StringRef WorkingDirectory, ArrayRef<ArrayRef<std::string>> CommandLines, DependencyConsumer &DepConsumer, DependencyActionController &Controller, - DiagnosticsEngine &Diags, + DiagnosticConsumer &DiagConsumer, llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS) { IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = nullptr; if (OverlayFS) { @@ -99,6 +94,11 @@ bool DependencyScanningWorker::computeDependencies( {Cmd.front(), {Cmd.begin() + 1, Cmd.end()}}); return true; } + + auto DiagEngineWithDiagOpts = + DiagnosticsEngineWithDiagOpts(Cmd, OverlayFS, DiagConsumer); + auto &Diags = *DiagEngineWithDiagOpts.DiagEngine; + // Create an invocation that uses the underlying file system to ensure that // any file system requests that are made by the driver do not go through // the dependency scanning filesystem. @@ -107,7 +107,7 @@ bool DependencyScanningWorker::computeDependencies( // Ensure finish() is called even if we never reached ExecuteAction(). if (!Action.hasDiagConsumerFinished()) - Diags.getClient()->finish(); + DiagConsumer.finish(); return Success && Action.hasScanned(); } diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp index 1aaa46c5266b1..079630e15d1d0 100644 --- a/clang/lib/Tooling/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanningTool.cpp @@ -136,16 +136,24 @@ buildFrontendCommandLine(const driver::Command &Cmd) { static bool computeDependenciesForDriverCommandLine( DependencyScanningWorker &Worker, StringRef WorkingDirectory, ArrayRef<std::string> CommandLine, DependencyConsumer &Consumer, - DependencyActionController &Controller, DiagnosticsEngine &Diags, + DependencyActionController &Controller, DiagnosticConsumer &DiagConsumer, IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS) { - Worker.getVFS().setCurrentWorkingDirectory(WorkingDirectory); + IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = nullptr; + if (OverlayFS) { + FS = OverlayFS; + } else { + FS = &Worker.getVFS(); + FS->setCurrentWorkingDirectory(WorkingDirectory); + } // Compilation holds a non-owning a reference to the Driver, hence we need to // keep the Driver alive when we use Compilation. Arguments to commands may be // owned by Alloc when expanded from response files. llvm::BumpPtrAllocator Alloc; - const auto [Driver, Compilation] = - buildCompilation(CommandLine, Diags, &Worker.getVFS(), Alloc); + auto DiagEngineWithDiagOpts = + DiagnosticsEngineWithDiagOpts(CommandLine, FS, DiagConsumer); + const auto [Driver, Compilation] = buildCompilation( + CommandLine, *DiagEngineWithDiagOpts.DiagEngine, FS, Alloc); if (!Compilation) return false; @@ -156,7 +164,8 @@ static bool computeDependenciesForDriverCommandLine( FrontendCommandLines.begin(), FrontendCommandLines.end()); return Worker.computeDependencies(WorkingDirectory, FrontendCommandLinesView, - Consumer, Controller, Diags, OverlayFS); + Consumer, Controller, DiagConsumer, + OverlayFS); } static llvm::Error makeErrorFromDiagnosticsOS( @@ -181,18 +190,13 @@ static bool computeDependencies( CommandLine = CommandLineWithTUBufferInput; } - DiagnosticsEngineWithDiagOpts DiagEngineWithDiagOpts( - CommandLine, &Worker.getVFS(), DiagConsumer); - auto &Diags = *DiagEngineWithDiagOpts.DiagEngine; - const auto IsCC1Input = (CommandLine.size() >= 2 && CommandLine[1] == "-cc1"); - return - IsCC1Input - ? Worker.computeDependencies(WorkingDirectory, CommandLine, Consumer, - Controller, Diags, OverlayFS) - : computeDependenciesForDriverCommandLine( - Worker, WorkingDirectory, CommandLine, Consumer, Controller, - Diags, OverlayFS); + return IsCC1Input ? Worker.computeDependencies(WorkingDirectory, CommandLine, + Consumer, Controller, + DiagConsumer, OverlayFS) + : computeDependenciesForDriverCommandLine( + Worker, WorkingDirectory, CommandLine, Consumer, + Controller, DiagConsumer, OverlayFS); } std::optional<std::string> >From aa5a43430b544eabf852e140567e3749256a1a49 Mon Sep 17 00:00:00 2001 From: Naveen Seth Hanig <[email protected]> Date: Tue, 16 Dec 2025 20:24:38 +0100 Subject: [PATCH 08/10] Formatting --- clang/lib/Tooling/DependencyScanningTool.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp index 079630e15d1d0..361c7b360f83f 100644 --- a/clang/lib/Tooling/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanningTool.cpp @@ -206,7 +206,7 @@ DependencyScanningTool::getDependencyFile(ArrayRef<std::string> CommandLine, MakeDependencyPrinterConsumer DepConsumer; CallbackActionController Controller(nullptr); if (!computeDependencies(Worker, CWD, CommandLine, DepConsumer, Controller, - DiagConsumer)) + DiagConsumer)) return std::nullopt; std::string Output; DepConsumer.printDependencies(Output); @@ -278,7 +278,7 @@ DependencyScanningTool::getTranslationUnitDependencies( CallbackActionController Controller(LookupModuleOutput); if (!computeDependencies(Worker, CWD, CommandLine, Consumer, Controller, - DiagConsumer, TUBuffer)) + DiagConsumer, TUBuffer)) return std::nullopt; return Consumer.takeTranslationUnitDeps(); } >From 4e1e9e5e57e1956d9b27133e1a83627ddb40e543 Mon Sep 17 00:00:00 2001 From: Naveen Seth Hanig <[email protected]> Date: Tue, 16 Dec 2025 22:10:28 +0100 Subject: [PATCH 09/10] Move TUBuffer handling to getTranslationUnitDependencies --- clang/lib/Tooling/DependencyScanningTool.cpp | 27 ++++++++++---------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp index 361c7b360f83f..062821e8242e7 100644 --- a/clang/lib/Tooling/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanningTool.cpp @@ -178,18 +178,8 @@ static bool computeDependencies( DependencyScanningWorker &Worker, StringRef WorkingDirectory, ArrayRef<std::string> CommandLine, DependencyConsumer &Consumer, DependencyActionController &Controller, DiagnosticConsumer &DiagConsumer, - std::optional<llvm::MemoryBufferRef> TUBuffer = std::nullopt) { - // If we are scanning from a TUBuffer, create an overlay filesystem with the - // input as an in-memory file and add it to the command line. - IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS = nullptr; - std::vector<std::string> CommandLineWithTUBufferInput; - if (TUBuffer) { - std::tie(OverlayFS, CommandLineWithTUBufferInput) = - initVFSForTUBufferScanning(&Worker.getVFS(), CommandLine, - WorkingDirectory, *TUBuffer); - CommandLine = CommandLineWithTUBufferInput; - } - + llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS = + nullptr) { const auto IsCC1Input = (CommandLine.size() >= 2 && CommandLine[1] == "-cc1"); return IsCC1Input ? Worker.computeDependencies(WorkingDirectory, CommandLine, Consumer, Controller, @@ -277,8 +267,19 @@ DependencyScanningTool::getTranslationUnitDependencies( FullDependencyConsumer Consumer(AlreadySeen); CallbackActionController Controller(LookupModuleOutput); + // If we are scanning from a TUBuffer, create an overlay filesystem with the + // input as an in-memory file and add it to the command line. + IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayFS = nullptr; + std::vector<std::string> CommandLineWithTUBufferInput; + if (TUBuffer) { + std::tie(OverlayFS, CommandLineWithTUBufferInput) = + initVFSForTUBufferScanning(&Worker.getVFS(), CommandLine, CWD, + *TUBuffer); + CommandLine = CommandLineWithTUBufferInput; + } + if (!computeDependencies(Worker, CWD, CommandLine, Consumer, Controller, - DiagConsumer, TUBuffer)) + DiagConsumer, OverlayFS)) return std::nullopt; return Consumer.takeTranslationUnitDeps(); } >From b19b2ea21525fa322498346fc02e72f67f661921 Mon Sep 17 00:00:00 2001 From: Naveen Seth Hanig <[email protected]> Date: Tue, 16 Dec 2025 22:10:54 +0100 Subject: [PATCH 10/10] Undo renaming to buildFrontendCommandLine --- clang/lib/Tooling/DependencyScanningTool.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp index 062821e8242e7..e61289f4a45d0 100644 --- a/clang/lib/Tooling/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanningTool.cpp @@ -124,7 +124,7 @@ buildCompilation(ArrayRef<std::string> ArgStrs, DiagnosticsEngine &Diags, /// Constructs the full frontend command line, including executable, for the /// given driver \c Cmd. static SmallVector<std::string, 0> -buildFrontendCommandLine(const driver::Command &Cmd) { +buildCC1CommandLine(const driver::Command &Cmd) { const auto &Args = Cmd.getArguments(); SmallVector<std::string, 0> Out; Out.reserve(Args.size() + 1); @@ -159,7 +159,7 @@ static bool computeDependenciesForDriverCommandLine( SmallVector<SmallVector<std::string, 0>> FrontendCommandLines; for (const auto &Cmd : Compilation->getJobs()) - FrontendCommandLines.push_back(buildFrontendCommandLine(Cmd)); + FrontendCommandLines.push_back(buildCC1CommandLine(Cmd)); SmallVector<ArrayRef<std::string>> FrontendCommandLinesView( FrontendCommandLines.begin(), FrontendCommandLines.end()); @@ -320,7 +320,7 @@ static std::optional<SmallVector<std::string, 0>> getFirstCC1CommandLine( const auto &Jobs = Compilation->getJobs(); if (const auto It = llvm::find_if(Jobs, IsClangCmd); It != Jobs.end()) - return buildFrontendCommandLine(*It); + return buildCC1CommandLine(*It); return std::nullopt; } _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
