ioeric created this revision.
ioeric added reviewers: sammccall, hokein.
Herald added subscribers: cfe-commits, kadircet, arphaman, jkorous, MaskRay,
ilya-biryukov.
This allows us to deduplicate header symbols across TUs. File digests
are collects when collecting symbols/refs. And the index store deduplicates
file symbols based on the file digest.
WIP: This is still a prototype and apparently needs better names and more tests.
Looking for early feedback to make sure this is heading the right direction.
Repository:
rCTE Clang Tools Extra
https://reviews.llvm.org/D53433
Files:
clangd/index/Background.cpp
clangd/index/Background.h
clangd/index/FileIndex.cpp
clangd/index/FileIndex.h
clangd/index/IndexAction.cpp
clangd/index/IndexAction.h
clangd/index/SymbolCollector.cpp
clangd/index/SymbolCollector.h
clangd/indexer/IndexerMain.cpp
unittests/clangd/BackgroundIndexTests.cpp
unittests/clangd/FileIndexTests.cpp
unittests/clangd/SyncAPI.cpp
unittests/clangd/SyncAPI.h
Index: unittests/clangd/SyncAPI.h
===================================================================
--- unittests/clangd/SyncAPI.h
+++ unittests/clangd/SyncAPI.h
@@ -52,6 +52,7 @@
SymbolSlab runFuzzyFind(const SymbolIndex &Index, StringRef Query);
SymbolSlab runFuzzyFind(const SymbolIndex &Index, const FuzzyFindRequest &Req);
+RefSlab getRefs(const SymbolIndex &Index, SymbolID ID);
} // namespace clangd
} // namespace clang
Index: unittests/clangd/SyncAPI.cpp
===================================================================
--- unittests/clangd/SyncAPI.cpp
+++ unittests/clangd/SyncAPI.cpp
@@ -8,6 +8,7 @@
//===----------------------------------------------------------------------===//
#include "SyncAPI.h"
+#include "index/Index.h"
namespace clang {
namespace clangd {
@@ -137,5 +138,14 @@
return std::move(Builder).build();
}
+RefSlab getRefs(const SymbolIndex &Index, SymbolID ID) {
+ RefsRequest Req;
+ Req.IDs = {ID};
+ RefSlab::Builder Slab;
+ Index.refs(Req, [&](const Ref &S) { Slab.insert(ID, S); });
+ return std::move(Slab).build();
+}
+
+
} // namespace clangd
} // namespace clang
Index: unittests/clangd/FileIndexTests.cpp
===================================================================
--- unittests/clangd/FileIndexTests.cpp
+++ unittests/clangd/FileIndexTests.cpp
@@ -72,14 +72,6 @@
return llvm::make_unique<RefSlab>(std::move(Slab).build());
}
-RefSlab getRefs(const SymbolIndex &I, SymbolID ID) {
- RefsRequest Req;
- Req.IDs = {ID};
- RefSlab::Builder Slab;
- I.refs(Req, [&](const Ref &S) { Slab.insert(ID, S); });
- return std::move(Slab).build();
-}
-
TEST(FileSymbolsTest, UpdateAndGet) {
FileSymbols FS;
EXPECT_THAT(runFuzzyFind(*FS.buildIndex(IndexType::Light), ""), IsEmpty());
Index: unittests/clangd/BackgroundIndexTests.cpp
===================================================================
--- unittests/clangd/BackgroundIndexTests.cpp
+++ unittests/clangd/BackgroundIndexTests.cpp
@@ -1,36 +1,78 @@
#include "SyncAPI.h"
#include "TestFS.h"
#include "index/Background.h"
+#include "gmock/gmock-generated-matchers.h"
+#include "gmock/gmock-matchers.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
+using testing::_;
+using testing::AllOf;
+using testing::Not;
using testing::UnorderedElementsAre;
namespace clang {
namespace clangd {
MATCHER_P(Named, N, "") { return arg.Name == N; }
+MATCHER(Declared, "") { return !arg.CanonicalDeclaration.FileURI.empty(); }
+MATCHER(Defined, "") { return !arg.Definition.FileURI.empty(); }
+
+MATCHER_P(FileURI, F, "") { return arg.Location.FileURI == F; }
+testing::Matcher<const RefSlab &>
+RefsAre(std::vector<testing::Matcher<Ref>> Matchers) {
+ return ElementsAre(testing::Pair(_, UnorderedElementsAreArray(Matchers)));
+}
TEST(BackgroundIndexTest, IndexTwoFiles) {
MockFSProvider FS;
// a.h yields different symbols when included by A.cc vs B.cc.
// Currently we store symbols for each TU, so we get both.
- FS.Files[testPath("root/A.h")] = "void a_h(); void NAME(){}";
- FS.Files[testPath("root/A.cc")] = "#include \"A.h\"";
- FS.Files[testPath("root/B.cc")] = "#define NAME bar\n#include \"A.h\"";
+ FS.Files[testPath("root/A.h")] = R"(
+ void common();
+ void f_b();
+ #if CC == A
+ class A_H {};
+ #elif CC == B
+ class B_H {};
+ #else
+ class _H {};
+ #endif
+ )";
+ FS.Files[testPath("root/A.cc")] =
+ "#include \"A.h\"\nvoid g() { (void)common; }";
+ FS.Files[testPath("root/B.cc")] =
+ "#define CC B\n#include \"A.h\"\nvoid f_b() { (void)common; }";
BackgroundIndex Idx(Context::empty(), "", FS);
tooling::CompileCommand Cmd;
Cmd.Filename = testPath("root/A.cc");
Cmd.Directory = testPath("root");
- Cmd.CommandLine = {"clang++", "-DNAME=foo", testPath("root/A.cc")};
+ Cmd.CommandLine = {"clang++", "-DCC=A", testPath("root/A.cc")};
Idx.enqueue(testPath("root"), Cmd);
- Cmd.CommandLine.back() = Cmd.Filename = testPath("root/B.cc");
+
+ Idx.blockUntilIdleForTest();
+ EXPECT_THAT(
+ runFuzzyFind(Idx, ""),
+ UnorderedElementsAre(Named("common"), Named("A_H"),
+ AllOf(Named("f_b"), Declared(), Not(Defined()))));
+
+ Cmd.Filename = testPath("root/B.cc");
+ Cmd.CommandLine = {"clang++", Cmd.Filename};
Idx.enqueue(testPath("root"), Cmd);
Idx.blockUntilIdleForTest();
+ // B_H is dropped as we don't collect symbols from A.h in this compilation.
EXPECT_THAT(runFuzzyFind(Idx, ""),
- UnorderedElementsAre(Named("a_h"), Named("foo"), Named("bar")));
+ UnorderedElementsAre(Named("common"), Named("A_H"),
+ AllOf(Named("f_b"), Declared(), Defined())));
+ auto Syms = runFuzzyFind(Idx, "common");
+ EXPECT_THAT(Syms, UnorderedElementsAre(Named("common")));
+ auto Common = *Syms.begin();
+ EXPECT_THAT(getRefs(Idx, Common.ID),
+ RefsAre({FileURI("file:///clangd-test/root/A.h"),
+ FileURI("file:///clangd-test/root/A.cc"),
+ FileURI("file:///clangd-test/root/B.cc")}));
}
} // namespace clangd
Index: clangd/indexer/IndexerMain.cpp
===================================================================
--- clangd/indexer/IndexerMain.cpp
+++ clangd/indexer/IndexerMain.cpp
@@ -63,7 +63,8 @@
for (const auto &Ref : Sym.second)
Refs.insert(Sym.first, Ref);
}
- })
+ },
+ nullptr)
.release();
}
Index: clangd/index/SymbolCollector.h
===================================================================
--- clangd/index/SymbolCollector.h
+++ clangd/index/SymbolCollector.h
@@ -10,12 +10,14 @@
#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_SYMBOL_COLLECTOR_H
#include "CanonicalIncludes.h"
+#include "FileIndex.h"
#include "Index.h"
#include "clang/AST/ASTContext.h"
#include "clang/AST/Decl.h"
#include "clang/Index/IndexDataConsumer.h"
#include "clang/Index/IndexSymbol.h"
#include "clang/Sema/CodeCompleteConsumer.h"
+#include "llvm/ADT/DenseSet.h"
namespace clang {
namespace clangd {
@@ -96,6 +98,7 @@
SymbolSlab takeSymbols() { return std::move(Symbols).build(); }
RefSlab takeRefs() { return std::move(Refs).build(); }
+ FileDigests takeFileDigests() { return std::move(IndexedFileDigests); }
void finish() override;
@@ -109,6 +112,7 @@
// Only symbols declared in preamble (from #include) and referenced from the
// main file will be included.
RefSlab::Builder Refs;
+ FileDigests IndexedFileDigests;
ASTContext *ASTCtx;
std::shared_ptr<Preprocessor> PP;
std::shared_ptr<GlobalCodeCompletionAllocator> CompletionAllocator;
Index: clangd/index/SymbolCollector.cpp
===================================================================
--- clangd/index/SymbolCollector.cpp
+++ clangd/index/SymbolCollector.cpp
@@ -24,10 +24,12 @@
#include "clang/Basic/Specifiers.h"
#include "clang/Index/IndexSymbol.h"
#include "clang/Index/USRGeneration.h"
+#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
+#include "llvm/Support/SHA1.h"
namespace clang {
namespace clangd {
@@ -202,18 +204,32 @@
CreatePosition(TokLoc.getLocWithOffset(TokenLength))};
}
+void digestFile(const SourceManager &SM, SourceLocation Loc,
+ llvm::StringRef FileURI, FileDigests *Digest) {
+ if (Digest->count(FileURI) != 0)
+ return;
+ FileID FID = SM.getFileID(Loc);
+ if (FID.isInvalid())
+ return;
+ bool Invalid = false;
+ StringRef Content = SM.getBufferData(FID, &Invalid);
+ if (!Invalid)
+ (*Digest)[FileURI] = digest(Content);
+}
+
// Return the symbol location of the token at \p TokLoc.
llvm::Optional<SymbolLocation>
getTokenLocation(SourceLocation TokLoc, const SourceManager &SM,
const SymbolCollector::Options &Opts,
- const clang::LangOptions &LangOpts,
+ const clang::LangOptions &LangOpts, FileDigests &Digest,
std::string &FileURIStorage) {
auto U = toURI(SM, SM.getFilename(TokLoc), Opts);
if (!U)
return llvm::None;
FileURIStorage = std::move(*U);
SymbolLocation Result;
Result.FileURI = FileURIStorage;
+ digestFile(SM, TokLoc, Result.FileURI, &Digest);
auto Range = getTokenRange(TokLoc, SM, LangOpts);
Result.Start = Range.first;
Result.End = Range.second;
@@ -425,8 +441,9 @@
S.Flags |= Symbol::IndexedForCodeCompletion;
S.SymInfo = index::getSymbolInfoForMacro(*MI);
std::string FileURI;
- if (auto DeclLoc = getTokenLocation(MI->getDefinitionLoc(), SM, Opts,
- PP->getLangOpts(), FileURI))
+ if (auto DeclLoc =
+ getTokenLocation(MI->getDefinitionLoc(), SM, Opts, PP->getLangOpts(),
+ IndexedFileDigests, FileURI))
S.CanonicalDeclaration = *DeclLoc;
CodeCompletionResult SymbolCompletion(Name);
@@ -503,6 +520,7 @@
for (const auto &LocAndRole : It.second) {
auto FileID = SM.getFileID(LocAndRole.first);
if (auto FileURI = GetURI(FileID)) {
+ digestFile(SM, LocAndRole.first, *FileURI, &IndexedFileDigests);
auto Range =
getTokenRange(LocAndRole.first, SM, ASTCtx->getLangOpts());
Ref R;
@@ -540,8 +558,9 @@
S.Flags |= Symbol::ImplementationDetail;
S.SymInfo = index::getSymbolInfo(&ND);
std::string FileURI;
- if (auto DeclLoc = getTokenLocation(findNameLoc(&ND), SM, Opts,
- ASTCtx->getLangOpts(), FileURI))
+ if (auto DeclLoc =
+ getTokenLocation(findNameLoc(&ND), SM, Opts, ASTCtx->getLangOpts(),
+ IndexedFileDigests, FileURI))
S.CanonicalDeclaration = *DeclLoc;
// Add completion info.
@@ -592,9 +611,9 @@
// in clang::index. We should only see one definition.
Symbol S = DeclSym;
std::string FileURI;
- if (auto DefLoc = getTokenLocation(findNameLoc(&ND),
- ND.getASTContext().getSourceManager(),
- Opts, ASTCtx->getLangOpts(), FileURI))
+ if (auto DefLoc = getTokenLocation(
+ findNameLoc(&ND), ND.getASTContext().getSourceManager(), Opts,
+ ASTCtx->getLangOpts(), IndexedFileDigests, FileURI))
S.Definition = *DefLoc;
Symbols.insert(S);
}
Index: clangd/index/IndexAction.h
===================================================================
--- clangd/index/IndexAction.h
+++ clangd/index/IndexAction.h
@@ -9,6 +9,7 @@
#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_INDEX_ACTION_H
#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_INDEX_ACTION_H
+#include "FileIndex.h"
#include "SymbolCollector.h"
#include "clang/Frontend/FrontendActions.h"
@@ -23,10 +24,11 @@
// - references are always counted
// - all references are collected (if RefsCallback is non-null)
// - the symbol origin is always Static
-std::unique_ptr<FrontendAction>
-createStaticIndexingAction(SymbolCollector::Options Opts,
- std::function<void(SymbolSlab)> SymbolsCallback,
- std::function<void(RefSlab)> RefsCallback);
+std::unique_ptr<FrontendAction> createStaticIndexingAction(
+ SymbolCollector::Options Opts,
+ std::function<void(SymbolSlab)> SymbolsCallback,
+ std::function<void(RefSlab)> RefsCallback,
+ std::function<void(FileDigests)> FileDigestsCallback);
} // namespace clangd
} // namespace clang
Index: clangd/index/IndexAction.cpp
===================================================================
--- clangd/index/IndexAction.cpp
+++ clangd/index/IndexAction.cpp
@@ -1,8 +1,10 @@
#include "IndexAction.h"
+#include "FileIndex.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Index/IndexDataConsumer.h"
#include "clang/Index/IndexingAction.h"
#include "clang/Tooling/Tooling.h"
+#include <functional>
namespace clang {
namespace clangd {
namespace {
@@ -14,10 +16,12 @@
std::unique_ptr<CanonicalIncludes> Includes,
const index::IndexingOptions &Opts,
std::function<void(SymbolSlab)> SymbolsCallback,
- std::function<void(RefSlab)> RefsCallback)
+ std::function<void(RefSlab)> RefsCallback,
+ std::function<void(FileDigests)> FileDigestsCallback)
: WrapperFrontendAction(index::createIndexingAction(C, Opts, nullptr)),
SymbolsCallback(SymbolsCallback), RefsCallback(RefsCallback),
- Collector(C), Includes(std::move(Includes)),
+ FileDigestsCallback(FileDigestsCallback), Collector(C),
+ Includes(std::move(Includes)),
PragmaHandler(collectIWYUHeaderMaps(this->Includes.get())) {}
std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
@@ -44,22 +48,26 @@
SymbolsCallback(Collector->takeSymbols());
if (RefsCallback != nullptr)
RefsCallback(Collector->takeRefs());
+ if (FileDigestsCallback)
+ FileDigestsCallback(Collector->takeFileDigests());
}
private:
std::function<void(SymbolSlab)> SymbolsCallback;
std::function<void(RefSlab)> RefsCallback;
+ std::function<void(FileDigests)> FileDigestsCallback;
std::shared_ptr<SymbolCollector> Collector;
std::unique_ptr<CanonicalIncludes> Includes;
std::unique_ptr<CommentHandler> PragmaHandler;
};
} // namespace
-std::unique_ptr<FrontendAction>
-createStaticIndexingAction(SymbolCollector::Options Opts,
- std::function<void(SymbolSlab)> SymbolsCallback,
- std::function<void(RefSlab)> RefsCallback) {
+std::unique_ptr<FrontendAction> createStaticIndexingAction(
+ SymbolCollector::Options Opts,
+ std::function<void(SymbolSlab)> SymbolsCallback,
+ std::function<void(RefSlab)> RefsCallback,
+ std::function<void(FileDigests)> FileDigestsCallback) {
index::IndexingOptions IndexOpts;
IndexOpts.SystemSymbolFilter =
index::IndexingOptions::SystemSymbolFilterKind::All;
@@ -75,7 +83,7 @@
Opts.Includes = Includes.get();
return llvm::make_unique<IndexAction>(
std::make_shared<SymbolCollector>(std::move(Opts)), std::move(Includes),
- IndexOpts, SymbolsCallback, RefsCallback);
+ IndexOpts, SymbolsCallback, RefsCallback, FileDigestsCallback);
}
} // namespace clangd
Index: clangd/index/FileIndex.h
===================================================================
--- clangd/index/FileIndex.h
+++ clangd/index/FileIndex.h
@@ -21,6 +21,9 @@
#include "MemIndex.h"
#include "Merge.h"
#include "clang/Lex/Preprocessor.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/SHA1.h"
#include <memory>
namespace clang {
@@ -34,6 +37,44 @@
Heavy,
};
+using FileDigest = decltype(llvm::SHA1::hash({}));
+using FileDigests = llvm::StringMap<FileDigest>;
+
+inline FileDigest digest(llvm::StringRef Content) {
+ return llvm::SHA1::hash({(const uint8_t *)Content.data(), Content.size()});
+}
+
+/// FIXME: rename to something else. FileSymbols is a good fit but taken. Maybe
+/// rename the existing FileSymbols to something else e.g. TUSymbols?
+class SymbolsGroupedByFiles {
+public:
+ /// Update with index results from indexing a TU. Index data is partitioned
+ /// into results and stores based on file. This only stores symbols/references
+ /// from files (in FileDigests) with new contents (i.e. new digest).
+ /// FIXME: make sure indexing a header file as main file and as included
+ /// header generate the same index results.
+ void update(SymbolSlab Symbols, RefSlab Refs, FileDigests Digests);
+
+ // Build an index on all symbols from all files.
+ std::unique_ptr<SymbolIndex>
+ buildIndex(IndexType, ArrayRef<std::string> URISchemes = {});
+
+private:
+ struct OneFileSymbols {
+ OneFileSymbols(llvm::StringRef FileURI, FileDigest Digest,
+ SymbolSlab Syms, RefSlab Refs)
+ : FileURI(FileURI), Digest(Digest), Symbols(std::move(Syms)),
+ Refs(std::move(Refs)) {}
+
+ llvm::StringRef FileURI;
+ const FileDigest Digest;
+ const SymbolSlab Symbols;
+ const RefSlab Refs;
+ };
+
+ llvm::StringMap<std::unique_ptr<OneFileSymbols>> AllSymbols;
+};
+
/// A container of Symbols from several source files. It can be updated
/// at source-file granularity, replacing all symbols from one file with a new
/// set.
@@ -52,7 +93,7 @@
/// Updates all symbols and refs in a file.
/// If either is nullptr, corresponding data for \p Path will be removed.
void update(PathRef Path, std::unique_ptr<SymbolSlab> Slab,
- std::unique_ptr<RefSlab> Refs);
+ std::unique_ptr<RefSlab> Ref);
// The index keeps the symbols alive.
std::unique_ptr<SymbolIndex>
@@ -65,6 +106,7 @@
llvm::StringMap<std::shared_ptr<SymbolSlab>> FileToSymbols;
/// Stores the latest ref snapshots for all active files.
llvm::StringMap<std::shared_ptr<RefSlab>> FileToRefs;
+ llvm::StringMap<FileDigests> Digests;
};
/// This manages symbols from files and an in-memory index on all symbols.
Index: clangd/index/FileIndex.cpp
===================================================================
--- clangd/index/FileIndex.cpp
+++ clangd/index/FileIndex.cpp
@@ -12,11 +12,16 @@
#include "Logger.h"
#include "SymbolCollector.h"
#include "index/Index.h"
+#include "index/MemIndex.h"
#include "index/Merge.h"
#include "index/dex/Dex.h"
#include "clang/Index/IndexingAction.h"
#include "clang/Lex/MacroInfo.h"
#include "clang/Lex/Preprocessor.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringRef.h"
#include <memory>
namespace clang {
@@ -86,6 +91,96 @@
.first;
}
+void SymbolsGroupedByFiles::update(SymbolSlab Symbols, RefSlab Refs,
+ FileDigests Digests) {
+ // Find files with new digests or not seen before.
+ llvm::DenseSet<llvm::StringRef> FilesToUpdate;
+ for (const auto &D : Digests) {
+ auto I = AllSymbols.find(D.first());
+ if (I == AllSymbols.end() || I->getValue()->Digest != D.getValue())
+ FilesToUpdate.insert(D.first());
+ }
+
+ // Partition symbols/references into files.
+ struct File {
+ llvm::DenseSet<const Symbol *> Symbols;
+ llvm::DenseSet<const Ref *> Refs;
+ };
+ llvm::StringMap<File> Files;
+ for (const auto &Sym : Symbols) {
+ if (FilesToUpdate.count(Sym.CanonicalDeclaration.FileURI) != 0) {
+ auto &F = Files[Sym.CanonicalDeclaration.FileURI];
+ F.Symbols.insert(&Sym);
+ }
+ if (Sym.Definition.FileURI != Sym.CanonicalDeclaration.FileURI &&
+ FilesToUpdate.count(Sym.Definition.FileURI) != 0) {
+ auto &F = Files[Sym.Definition.FileURI];
+ F.Symbols.insert(&Sym);
+ }
+ }
+ llvm::DenseMap<const Ref *, SymbolID> RefToIDs;
+ for (const auto &SymRefs : Refs) {
+ for (const auto &R : SymRefs.second) {
+ if (FilesToUpdate.count(R.Location.FileURI) != 0) {
+ auto &F = Files[R.Location.FileURI];
+ RefToIDs[&R] = SymRefs.first;
+ F.Refs.insert(&R);
+ }
+ }
+ }
+
+ // Build and store new slabs for each updated file.
+ for (const auto &F : Files) {
+ log("> update symbols in {0}", F.first());
+ StringRef FileURI = F.getKey();
+ SymbolSlab::Builder Syms;
+ RefSlab::Builder Refs;
+ for (const auto *S : F.getValue().Symbols) {
+ Syms.insert(*S);
+ log(" add sym: {0}{1}, {2}", S->Scope, S->Name,
+ S->CanonicalDeclaration.FileURI);
+ }
+ for (const auto *R : F.getValue().Refs) {
+ Refs.insert(RefToIDs[R], *R);
+ log(" add ref: {0} at {1}", RefToIDs[R], R->Location);
+ }
+ AllSymbols[FileURI] = llvm::make_unique<OneFileSymbols>(
+ FileURI, Digests[FileURI], std::move(Syms).build(),
+ std::move(Refs).build());
+ }
+}
+
+std::unique_ptr<SymbolIndex>
+SymbolsGroupedByFiles::buildIndex(IndexType Type,
+ ArrayRef<std::string> URISchemes) {
+ SymbolSlab::Builder Syms;
+ RefSlab::Builder Refs;
+ for (const auto &FileSyms : AllSymbols) {
+ for (const auto &Sym : FileSyms.getValue()->Symbols) {
+ // FIXME: add comment explaining why this works for fwd decls vs actual
+ // decls.
+ if (const auto *Existing = Syms.find(Sym.ID))
+ Syms.insert(mergeSymbol(*Existing, Sym));
+ else
+ Syms.insert(Sym);
+ }
+ // FIXME: aggregate symbol reference count based on references.
+ for (const auto &Sym : FileSyms.getValue()->Refs) {
+ for (const auto &Ref : Sym.second)
+ Refs.insert(Sym.first, Ref);
+ }
+ }
+ auto Symbols = std::move(Syms).build();
+ auto References = std::move(Refs).build();
+ switch (Type) {
+ case IndexType::Light:
+ return MemIndex::build(std::move(Symbols), std::move(References));
+ case IndexType::Heavy:
+ return dex::Dex::build(std::move(Symbols), std::move(References),
+ URISchemes);
+ }
+}
+
void FileSymbols::update(PathRef Path, std::unique_ptr<SymbolSlab> Symbols,
std::unique_ptr<RefSlab> Refs) {
std::lock_guard<std::mutex> Lock(Mutex);
Index: clangd/index/Background.h
===================================================================
--- clangd/index/Background.h
+++ clangd/index/Background.h
@@ -15,6 +15,7 @@
#include "index/FileIndex.h"
#include "index/Index.h"
#include "clang/Tooling/CompilationDatabase.h"
+#include "llvm/ADT/StringMap.h"
#include "llvm/Support/SHA1.h"
#include <condition_variable>
#include <deque>
@@ -27,6 +28,7 @@
// all commands in a compilation database. Indexing happens in the background.
// FIXME: it should also persist its state on disk for fast start.
// FIXME: it should watch for changes to files on disk.
+// FIXME: support different URI schemes.
class BackgroundIndex : public SwapIndex {
public:
// FIXME: resource-dir injection should be hoisted somewhere common.
@@ -57,9 +59,9 @@
// index state
llvm::Error index(tooling::CompileCommand);
- FileSymbols IndexedSymbols; // Index contents.
- using Hash = decltype(llvm::SHA1::hash({}));
- llvm::StringMap<Hash> FileHash; // Digest of indexed file.
+
+ SymbolsGroupedByFiles IndexedSymbols;
+ FileDigests MainFileDigests;
// queue management
using Task = std::function<void()>; // FIXME: use multiple worker threads.
Index: clangd/index/Background.cpp
===================================================================
--- clangd/index/Background.cpp
+++ clangd/index/Background.cpp
@@ -126,10 +126,9 @@
auto Buf = FS->getBufferForFile(AbsolutePath);
if (!Buf)
return errorCodeToError(Buf.getError());
- StringRef Contents = Buf->get()->getBuffer();
- auto Hash = SHA1::hash({(const uint8_t *)Contents.data(), Contents.size()});
+ auto Hash = digest(Buf->get()->getBuffer());
- if (FileHash.lookup(AbsolutePath) == Hash) {
+ if (MainFileDigests.lookup(AbsolutePath) == Hash) {
vlog("No need to index {0}, already up to date", AbsolutePath);
return Error::success();
}
@@ -154,10 +153,12 @@
SymbolCollector::Options IndexOpts;
SymbolSlab Symbols;
RefSlab Refs;
+ FileDigests Digests;
IndexFileIn IndexData;
auto Action = createStaticIndexingAction(
IndexOpts, [&](SymbolSlab S) { Symbols = std::move(S); },
- [&](RefSlab R) { Refs = std::move(R); });
+ [&](RefSlab R) { Refs = std::move(R); },
+ [&](FileDigests D) { Digests = std::move(D); });
// We're going to run clang here, and it could potentially crash.
// We could use CrashRecoveryContext to try to make indexing crashes nonfatal,
@@ -177,11 +178,9 @@
Symbols.size(), Refs.size());
SPAN_ATTACH(Tracer, "symbols", int(Symbols.size()));
SPAN_ATTACH(Tracer, "refs", int(Refs.size()));
- // FIXME: partition the symbols by file rather than TU, to avoid duplication.
- IndexedSymbols.update(AbsolutePath,
- llvm::make_unique<SymbolSlab>(std::move(Symbols)),
- llvm::make_unique<RefSlab>(std::move(Refs)));
- FileHash[AbsolutePath] = Hash;
+ IndexedSymbols.update(std::move(Symbols), std::move(Refs),
+ std::move(Digests));
+ MainFileDigests[AbsolutePath] = Hash;
// FIXME: this should rebuild once-in-a-while, not after every file.
// At that point we should use Dex, too.
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
