ilya-biryukov created this revision.
ilya-biryukov added reviewers: hokein, sammccall, klimek.
Herald added subscribers: MaskRay, ioeric, jkorous-apple.
This avoids storing intermediate symbols in memory, most of which are
duplicates.
The resulting .yaml file is ~120MB, while intermediate symbols takes
more than 20GB.
Repository:
rCTE Clang Tools Extra
https://reviews.llvm.org/D45478
Files:
clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp
Index: clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp
===================================================================
--- clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp
+++ clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp
@@ -31,6 +31,7 @@
#include "llvm/Support/Signals.h"
#include "llvm/Support/ThreadPool.h"
#include "llvm/Support/YAMLTraits.h"
+#include <mutex>
using namespace llvm;
using namespace clang::tooling;
@@ -49,22 +50,47 @@
"not given, such headers will have relative paths."),
llvm::cl::init(""));
+/// Combines occurrences of the same symbols across translation units.
+class SymbolMerger {
+public:
+ void mergeSymbols(const SymbolSlab &Symbols) {
+ std::lock_guard<std::mutex> Lock(Mut);
+ for (const Symbol &Sym : Symbols) {
+ if (const auto *Existing = UniqueSymbols.find(Sym.ID)) {
+ Symbol::Details Scratch;
+ UniqueSymbols.insert(mergeSymbol(*Existing, Sym, &Scratch));
+ } else {
+ UniqueSymbols.insert(Sym);
+ }
+ }
+ }
+
+ SymbolSlab build() && {
+ std::lock_guard<std::mutex> Lock(Mut);
+ return std::move(UniqueSymbols).build();
+ }
+
+private:
+ std::mutex Mut;
+ SymbolSlab::Builder UniqueSymbols;
+};
+
class SymbolIndexActionFactory : public tooling::FrontendActionFactory {
public:
- SymbolIndexActionFactory(tooling::ExecutionContext *Ctx) : Ctx(Ctx) {}
+ SymbolIndexActionFactory(SymbolMerger &Merger) : Merger(Merger) {}
clang::FrontendAction *create() override {
// Wraps the index action and reports collected symbols to the execution
// context at the end of each translation unit.
class WrappedIndexAction : public WrapperFrontendAction {
public:
- WrappedIndexAction(std::shared_ptr<SymbolCollector> C,
+ WrappedIndexAction(SymbolMerger &Merger,
+ std::shared_ptr<SymbolCollector> C,
std::unique_ptr<CanonicalIncludes> Includes,
- const index::IndexingOptions &Opts,
- tooling::ExecutionContext *Ctx)
+ const index::IndexingOptions &Opts)
: WrapperFrontendAction(
index::createIndexingAction(C, Opts, nullptr)),
- Ctx(Ctx), Collector(C), Includes(std::move(Includes)),
+ Merger(Merger), Collector(C), Includes(std::move(Includes)),
PragmaHandler(collectIWYUHeaderMaps(this->Includes.get())) {}
std::unique_ptr<ASTConsumer>
@@ -76,17 +102,11 @@
void EndSourceFileAction() override {
WrapperFrontendAction::EndSourceFileAction();
- auto Symbols = Collector->takeSymbols();
- for (const auto &Sym : Symbols) {
- std::string IDStr;
- llvm::raw_string_ostream OS(IDStr);
- OS << Sym.ID;
- Ctx->reportResult(OS.str(), SymbolToYAML(Sym));
- }
+ Merger.mergeSymbols(Collector->takeSymbols());
}
private:
- tooling::ExecutionContext *Ctx;
+ SymbolMerger &Merger;
std::shared_ptr<SymbolCollector> Collector;
std::unique_ptr<CanonicalIncludes> Includes;
std::unique_ptr<CommentHandler> PragmaHandler;
@@ -104,32 +124,13 @@
addSystemHeadersMapping(Includes.get());
CollectorOpts.Includes = Includes.get();
return new WrappedIndexAction(
- std::make_shared<SymbolCollector>(std::move(CollectorOpts)),
- std::move(Includes), IndexOpts, Ctx);
+ Merger, std::make_shared<SymbolCollector>(std::move(CollectorOpts)),
+ std::move(Includes), IndexOpts);
}
- tooling::ExecutionContext *Ctx;
+ SymbolMerger &Merger;
};
-// Combine occurrences of the same symbol across translation units.
-SymbolSlab mergeSymbols(tooling::ToolResults *Results) {
- SymbolSlab::Builder UniqueSymbols;
- llvm::BumpPtrAllocator Arena;
- Symbol::Details Scratch;
- Results->forEachResult([&](llvm::StringRef Key, llvm::StringRef Value) {
- Arena.Reset();
- llvm::yaml::Input Yin(Value, &Arena);
- auto Sym = clang::clangd::SymbolFromYAML(Yin, Arena);
- clang::clangd::SymbolID ID;
- Key >> ID;
- if (const auto *Existing = UniqueSymbols.find(ID))
- UniqueSymbols.insert(mergeSymbol(*Existing, Sym, &Scratch));
- else
- UniqueSymbols.insert(Sym);
- });
- return std::move(UniqueSymbols).build();
-}
-
} // namespace
} // namespace clangd
} // namespace clang
@@ -155,18 +156,14 @@
return 1;
}
- // Map phase: emit symbols found in each translation unit.
+ clang::clangd::SymbolMerger Merger;
+ // Emit and merge symbols found in each translation unit.
auto Err = Executor->get()->execute(
- llvm::make_unique<clang::clangd::SymbolIndexActionFactory>(
- Executor->get()->getExecutionContext()));
- if (Err) {
+ llvm::make_unique<clang::clangd::SymbolIndexActionFactory>(Merger));
+ if (Err)
llvm::errs() << llvm::toString(std::move(Err)) << "\n";
- }
-
- // Reduce phase: combine symbols using the ID as a key.
- auto UniqueSymbols =
- clang::clangd::mergeSymbols(Executor->get()->getToolResults());
-
+ // Get resulting symbols.
+ auto UniqueSymbols = std::move(Merger).build();
// Output phase: emit YAML for result symbols.
SymbolsToYAML(UniqueSymbols, llvm::outs());
return 0;
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
