ioeric updated this revision to Diff 170664.
ioeric marked 5 inline comments as done.
ioeric added a comment.
- address review comments
- merged with origin/master
- Cleanup
Repository:
rCTE Clang Tools Extra
https://reviews.llvm.org/D53433
Files:
clangd/index/Background.cpp
clangd/index/Background.h
clangd/index/FileIndex.cpp
clangd/index/FileIndex.h
clangd/index/SymbolCollector.cpp
clangd/index/SymbolCollector.h
unittests/clangd/BackgroundIndexTests.cpp
unittests/clangd/FileIndexTests.cpp
unittests/clangd/SyncAPI.cpp
unittests/clangd/SyncAPI.h
Index: unittests/clangd/SyncAPI.h
===================================================================
--- unittests/clangd/SyncAPI.h
+++ unittests/clangd/SyncAPI.h
@@ -52,6 +52,7 @@
SymbolSlab runFuzzyFind(const SymbolIndex &Index, StringRef Query);
SymbolSlab runFuzzyFind(const SymbolIndex &Index, const FuzzyFindRequest &Req);
+RefSlab getRefs(const SymbolIndex &Index, SymbolID ID);
} // namespace clangd
} // namespace clang
Index: unittests/clangd/SyncAPI.cpp
===================================================================
--- unittests/clangd/SyncAPI.cpp
+++ unittests/clangd/SyncAPI.cpp
@@ -8,6 +8,7 @@
//===----------------------------------------------------------------------===//
#include "SyncAPI.h"
+#include "index/Index.h"
using namespace llvm;
namespace clang {
@@ -138,5 +139,14 @@
return std::move(Builder).build();
}
+RefSlab getRefs(const SymbolIndex &Index, SymbolID ID) {
+ RefsRequest Req;
+ Req.IDs = {ID};
+ RefSlab::Builder Slab;
+ Index.refs(Req, [&](const Ref &S) { Slab.insert(ID, S); });
+ return std::move(Slab).build();
+}
+
+
} // namespace clangd
} // namespace clang
Index: unittests/clangd/FileIndexTests.cpp
===================================================================
--- unittests/clangd/FileIndexTests.cpp
+++ unittests/clangd/FileIndexTests.cpp
@@ -14,6 +14,7 @@
#include "TestFS.h"
#include "TestTU.h"
#include "index/FileIndex.h"
+#include "index/Index.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Frontend/PCHContainerOperations.h"
#include "clang/Frontend/Utils.h"
@@ -39,6 +40,7 @@
}
MATCHER_P(FileURI, F, "") { return arg.Location.FileURI == F; }
MATCHER_P(DeclURI, U, "") { return arg.CanonicalDeclaration.FileURI == U; }
+MATCHER_P(DefURI, U, "") { return arg.Definition.FileURI == U; }
MATCHER_P(QName, N, "") { return (arg.Scope + arg.Name).str() == N; }
using namespace llvm;
@@ -73,14 +75,6 @@
return llvm::make_unique<RefSlab>(std::move(Slab).build());
}
-RefSlab getRefs(const SymbolIndex &I, SymbolID ID) {
- RefsRequest Req;
- Req.IDs = {ID};
- RefSlab::Builder Slab;
- I.refs(Req, [&](const Ref &S) { Slab.insert(ID, S); });
- return std::move(Slab).build();
-}
-
TEST(FileSymbolsTest, UpdateAndGet) {
FileSymbols FS;
EXPECT_THAT(runFuzzyFind(*FS.buildIndex(IndexType::Light), ""), IsEmpty());
@@ -102,6 +96,27 @@
QName("4"), QName("5")));
}
+TEST(FileSymbolsTest, MergeOverlap) {
+ FileSymbols FS;
+ auto OneSymboSlab = [](Symbol Sym) {
+ SymbolSlab::Builder S;
+ S.insert(Sym);
+ return make_unique<SymbolSlab>(std::move(S).build());
+ };
+ auto X1 = symbol("x");
+ X1.CanonicalDeclaration.FileURI = "file:///x1";
+ auto X2 = symbol("x");
+ X2.Definition.FileURI = "file:///x2";
+
+ FS.update("f1", OneSymboSlab(X1), nullptr);
+ FS.update("f2", OneSymboSlab(X2), nullptr);
+ for (auto Type : {IndexType::Light, IndexType::Heavy})
+ EXPECT_THAT(
+ runFuzzyFind(*FS.buildIndex(Type, DuplicateHandling::Merge), "x"),
+ UnorderedElementsAre(
+ AllOf(QName("x"), DeclURI("file:///x1"), DefURI("file:///x2"))));
+}
+
TEST(FileSymbolsTest, SnapshotAliveAfterRemove) {
FileSymbols FS;
Index: unittests/clangd/BackgroundIndexTests.cpp
===================================================================
--- unittests/clangd/BackgroundIndexTests.cpp
+++ unittests/clangd/BackgroundIndexTests.cpp
@@ -4,33 +4,72 @@
#include "gmock/gmock.h"
#include "gtest/gtest.h"
+using testing::_;
+using testing::AllOf;
+using testing::Not;
using testing::UnorderedElementsAre;
namespace clang {
namespace clangd {
MATCHER_P(Named, N, "") { return arg.Name == N; }
+MATCHER(Declared, "") { return !arg.CanonicalDeclaration.FileURI.empty(); }
+MATCHER(Defined, "") { return !arg.Definition.FileURI.empty(); }
+
+MATCHER_P(FileURI, F, "") { return arg.Location.FileURI == F; }
+testing::Matcher<const RefSlab &>
+RefsAre(std::vector<testing::Matcher<Ref>> Matchers) {
+ return ElementsAre(testing::Pair(_, UnorderedElementsAreArray(Matchers)));
+}
TEST(BackgroundIndexTest, IndexTwoFiles) {
MockFSProvider FS;
// a.h yields different symbols when included by A.cc vs B.cc.
// Currently we store symbols for each TU, so we get both.
- FS.Files[testPath("root/A.h")] = "void a_h(); void NAME(){}";
- FS.Files[testPath("root/A.cc")] = "#include \"A.h\"";
- FS.Files[testPath("root/B.cc")] = "#define NAME bar\n#include \"A.h\"";
- BackgroundIndex Idx(Context::empty(), "", FS);
+ FS.Files[testPath("root/A.h")] = R"(
+ void common();
+ void f_b();
+ #if A
+ class A_H {};
+ #else
+ class B_H {};
+ #endif
+ )";
+ FS.Files[testPath("root/A.cc")] =
+ "#include \"A.h\"\nvoid g() { (void)common; }";
+ FS.Files[testPath("root/B.cc")] =
+ "#define A 0\n#include \"A.h\"\nvoid f_b() { (void)common; }";
+ BackgroundIndex Idx(Context::empty(), "", FS, /*URISchmes=*/{"unittest"});
tooling::CompileCommand Cmd;
Cmd.Filename = testPath("root/A.cc");
Cmd.Directory = testPath("root");
- Cmd.CommandLine = {"clang++", "-DNAME=foo", testPath("root/A.cc")};
+ Cmd.CommandLine = {"clang++", "-DA=1", testPath("root/A.cc")};
Idx.enqueue(testPath("root"), Cmd);
- Cmd.CommandLine.back() = Cmd.Filename = testPath("root/B.cc");
+
+ Idx.blockUntilIdleForTest();
+ EXPECT_THAT(
+ runFuzzyFind(Idx, ""),
+ UnorderedElementsAre(Named("common"), Named("A_H"),
+ AllOf(Named("f_b"), Declared(), Not(Defined()))));
+
+ Cmd.Filename = testPath("root/B.cc");
+ Cmd.CommandLine = {"clang++", Cmd.Filename};
Idx.enqueue(testPath("root"), Cmd);
Idx.blockUntilIdleForTest();
+ // B_H is dropped as we don't collect symbols from A.h in this compilation.
EXPECT_THAT(runFuzzyFind(Idx, ""),
- UnorderedElementsAre(Named("a_h"), Named("foo"), Named("bar")));
+ UnorderedElementsAre(Named("common"), Named("A_H"),
+ AllOf(Named("f_b"), Declared(), Defined())));
+
+ auto Syms = runFuzzyFind(Idx, "common");
+ EXPECT_THAT(Syms, UnorderedElementsAre(Named("common")));
+ auto Common = *Syms.begin();
+ EXPECT_THAT(getRefs(Idx, Common.ID),
+ RefsAre({FileURI("unittest:///root/A.h"),
+ FileURI("unittest:///root/A.cc"),
+ FileURI("unittest:///root/B.cc")}));
}
} // namespace clangd
Index: clangd/index/SymbolCollector.h
===================================================================
--- clangd/index/SymbolCollector.h
+++ clangd/index/SymbolCollector.h
@@ -13,9 +13,13 @@
#include "Index.h"
#include "clang/AST/ASTContext.h"
#include "clang/AST/Decl.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/SourceManager.h"
#include "clang/Index/IndexDataConsumer.h"
#include "clang/Index/IndexSymbol.h"
#include "clang/Sema/CodeCompleteConsumer.h"
+#include "llvm/ADT/DenseMap.h"
+#include <functional>
namespace clang {
namespace clangd {
@@ -69,6 +73,9 @@
/// collect macros. For example, `indexTopLevelDecls` will not index any
/// macro even if this is true.
bool CollectMacro = false;
+ /// If this is set, only collect symbols/references from a file if
+ /// `FileFilter(SM, FID)` is true. If not set, all files are indexed.
+ std::function<bool(const SourceManager &, FileID)> FileFilter = nullptr;
};
SymbolCollector(Options Opts);
@@ -125,6 +132,8 @@
// canonical by clang but should not be considered canonical in the index
// unless it's a definition.
llvm::DenseMap<const Decl *, const Decl *> CanonicalDecls;
+ // Cache whether to index a file or not.
+ llvm::DenseMap<FileID, bool> FilesToIndexCache;
};
} // namespace clangd
Index: clangd/index/SymbolCollector.cpp
===================================================================
--- clangd/index/SymbolCollector.cpp
+++ clangd/index/SymbolCollector.cpp
@@ -203,6 +203,18 @@
CreatePosition(TokLoc.getLocWithOffset(TokenLength))};
}
+bool shouldIndexFile(const SourceManager &SM, FileID FID,
+ const SymbolCollector::Options &Opts,
+ llvm::DenseMap<FileID, bool> *FilesToIndexCache) {
+ if (!Opts.FileFilter)
+ return true;
+ auto I = FilesToIndexCache->try_emplace(FID);
+ if (!I.second)
+ return I.first->second;
+ I.first->second = Opts.FileFilter(SM, FID);
+ return I.first->second;
+}
+
// Return the symbol location of the token at \p TokLoc.
Optional<SymbolLocation> getTokenLocation(SourceLocation TokLoc,
const SourceManager &SM,
@@ -426,8 +438,11 @@
S.Flags |= Symbol::IndexedForCodeCompletion;
S.SymInfo = index::getSymbolInfoForMacro(*MI);
std::string FileURI;
- if (auto DeclLoc = getTokenLocation(MI->getDefinitionLoc(), SM, Opts,
- PP->getLangOpts(), FileURI))
+ auto DefLoc = MI->getDefinitionLoc();
+ // FIXME: use the result to filter out symbols.
+ shouldIndexFile(SM, SM.getFileID(Loc), Opts, &FilesToIndexCache);
+ if (auto DeclLoc =
+ getTokenLocation(DefLoc, SM, Opts, PP->getLangOpts(), FileURI))
S.CanonicalDeclaration = *DeclLoc;
CodeCompletionResult SymbolCompletion(Name);
@@ -503,6 +518,8 @@
if (auto ID = getSymbolID(It.first)) {
for (const auto &LocAndRole : It.second) {
auto FileID = SM.getFileID(LocAndRole.first);
+ // FIXME: use the result to filter out references.
+ shouldIndexFile(SM, FileID, Opts, &FilesToIndexCache);
if (auto FileURI = GetURI(FileID)) {
auto Range =
getTokenRange(LocAndRole.first, SM, ASTCtx->getLangOpts());
@@ -521,6 +538,7 @@
ReferencedDecls.clear();
ReferencedMacros.clear();
DeclRefs.clear();
+ FilesToIndexCache.clear();
}
const Symbol *SymbolCollector::addDeclaration(const NamedDecl &ND,
@@ -541,8 +559,11 @@
S.Flags |= Symbol::ImplementationDetail;
S.SymInfo = index::getSymbolInfo(&ND);
std::string FileURI;
- if (auto DeclLoc = getTokenLocation(findNameLoc(&ND), SM, Opts,
- ASTCtx->getLangOpts(), FileURI))
+ auto Loc = findNameLoc(&ND);
+ // FIXME: use the result to filter out symbols.
+ shouldIndexFile(SM, SM.getFileID(Loc), Opts, &FilesToIndexCache);
+ if (auto DeclLoc =
+ getTokenLocation(Loc, SM, Opts, ASTCtx->getLangOpts(), FileURI))
S.CanonicalDeclaration = *DeclLoc;
// Add completion info.
@@ -593,8 +614,11 @@
// in clang::index. We should only see one definition.
Symbol S = DeclSym;
std::string FileURI;
- if (auto DefLoc = getTokenLocation(findNameLoc(&ND),
- ND.getASTContext().getSourceManager(),
+ auto Loc = findNameLoc(&ND);
+ const auto &SM = ND.getASTContext().getSourceManager();
+ // FIXME: use the result to filter out symbols.
+ shouldIndexFile(SM, SM.getFileID(Loc), Opts, &FilesToIndexCache);
+ if (auto DefLoc = getTokenLocation(Loc, ND.getASTContext().getSourceManager(),
Opts, ASTCtx->getLangOpts(), FileURI))
S.Definition = *DefLoc;
Symbols.insert(S);
Index: clangd/index/FileIndex.h
===================================================================
--- clangd/index/FileIndex.h
+++ clangd/index/FileIndex.h
@@ -34,6 +34,14 @@
Heavy,
};
+/// How to handle duplicated symbols across multiple files.
+enum class DuplicateHandling {
+ // Pick a random symbol. Less accurate but faster.
+ PickOne,
+ // Merge symbols. More accurate but slower.
+ Merge,
+};
+
/// A container of Symbols from several source files. It can be updated
/// at source-file granularity, replacing all symbols from one file with a new
/// set.
@@ -56,7 +64,9 @@
// The index keeps the symbols alive.
std::unique_ptr<SymbolIndex>
- buildIndex(IndexType, ArrayRef<std::string> URISchemes = {});
+ buildIndex(IndexType,
+ DuplicateHandling DuplicateHandle = DuplicateHandling::PickOne,
+ ArrayRef<std::string> URISchemes = {});
private:
mutable std::mutex Mutex;
Index: clangd/index/FileIndex.cpp
===================================================================
--- clangd/index/FileIndex.cpp
+++ clangd/index/FileIndex.cpp
@@ -12,11 +12,15 @@
#include "Logger.h"
#include "SymbolCollector.h"
#include "index/Index.h"
+#include "index/MemIndex.h"
#include "index/Merge.h"
#include "index/dex/Dex.h"
#include "clang/Index/IndexingAction.h"
#include "clang/Lex/MacroInfo.h"
#include "clang/Lex/Preprocessor.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringRef.h"
#include <memory>
using namespace llvm;
@@ -101,7 +105,8 @@
}
std::unique_ptr<SymbolIndex>
-FileSymbols::buildIndex(IndexType Type, ArrayRef<std::string> URISchemes) {
+FileSymbols::buildIndex(IndexType Type, DuplicateHandling DuplicateHandle,
+ ArrayRef<std::string> URISchemes) {
std::vector<std::shared_ptr<SymbolSlab>> SymbolSlabs;
std::vector<std::shared_ptr<RefSlab>> RefSlabs;
{
@@ -112,9 +117,33 @@
RefSlabs.push_back(FileAndRefs.second);
}
std::vector<const Symbol *> AllSymbols;
- for (const auto &Slab : SymbolSlabs)
- for (const auto &Sym : *Slab)
- AllSymbols.push_back(&Sym);
+ std::vector<Symbol> SymsStorage;
+ switch (DuplicateHandle) {
+ case DuplicateHandling::Merge: {
+ // Merge slabs into a single slab and only keep alive the merged.
+ DenseMap<SymbolID, Symbol> Merged;
+ for (const auto &Slab : SymbolSlabs) {
+ for (const auto &Sym : *Slab) {
+ auto I = Merged.try_emplace(Sym.ID, Sym);
+ if (!I.second)
+ I.first->second = mergeSymbol(std::move(I.first->second), Sym);
+ }
+ }
+ SymsStorage.reserve(Merged.size());
+ for (auto &Sym : Merged) {
+ SymsStorage.push_back(std::move(Sym.second));
+ AllSymbols.push_back(&SymsStorage.back());
+ }
+ // FIXME: aggregate symbol reference count based on references.
+ break;
+ }
+ case DuplicateHandling::PickOne: {
+ for (const auto &Slab : SymbolSlabs)
+ for (const auto &Sym : *Slab)
+ AllSymbols.push_back(&Sym);
+ break;
+ }
+ }
std::vector<Ref> RefsStorage; // Contiguous ranges for each SymbolID.
DenseMap<SymbolID, ArrayRef<Ref>> AllRefs;
@@ -140,7 +169,8 @@
}
}
- size_t StorageSize = RefsStorage.size() * sizeof(Ref);
+ size_t StorageSize =
+ RefsStorage.size() * sizeof(Ref) + SymsStorage.size() * sizeof(Symbol);
for (const auto &Slab : SymbolSlabs)
StorageSize += Slab->bytes();
for (const auto &RefSlab : RefSlabs)
@@ -152,13 +182,13 @@
return llvm::make_unique<MemIndex>(
make_pointee_range(AllSymbols), std::move(AllRefs),
std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs),
- std::move(RefsStorage)),
+ std::move(RefsStorage), std::move(SymsStorage)),
StorageSize);
case IndexType::Heavy:
return llvm::make_unique<dex::Dex>(
make_pointee_range(AllSymbols), std::move(AllRefs),
std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs),
- std::move(RefsStorage)),
+ std::move(RefsStorage), std::move(SymsStorage)),
StorageSize, std::move(URISchemes));
}
llvm_unreachable("Unknown clangd::IndexType");
@@ -176,16 +206,18 @@
PreambleSymbols.update(Path,
llvm::make_unique<SymbolSlab>(std::move(Symbols)),
llvm::make_unique<RefSlab>());
- PreambleIndex.reset(PreambleSymbols.buildIndex(
- UseDex ? IndexType::Heavy : IndexType::Light, URISchemes));
+ PreambleIndex.reset(
+ PreambleSymbols.buildIndex(UseDex ? IndexType::Heavy : IndexType::Light,
+ DuplicateHandling::PickOne, URISchemes));
}
void FileIndex::updateMain(PathRef Path, ParsedAST &AST) {
auto Contents = indexMainDecls(AST, URISchemes);
MainFileSymbols.update(
Path, llvm::make_unique<SymbolSlab>(std::move(Contents.first)),
llvm::make_unique<RefSlab>(std::move(Contents.second)));
- MainFileIndex.reset(MainFileSymbols.buildIndex(IndexType::Light, URISchemes));
+ MainFileIndex.reset(MainFileSymbols.buildIndex(
+ IndexType::Light, DuplicateHandling::PickOne, URISchemes));
}
} // namespace clangd
Index: clangd/index/Background.h
===================================================================
--- clangd/index/Background.h
+++ clangd/index/Background.h
@@ -15,6 +15,7 @@
#include "index/FileIndex.h"
#include "index/Index.h"
#include "clang/Tooling/CompilationDatabase.h"
+#include "llvm/ADT/StringMap.h"
#include "llvm/Support/SHA1.h"
#include <condition_variable>
#include <deque>
@@ -33,8 +34,7 @@
public:
// FIXME: resource-dir injection should be hoisted somewhere common.
BackgroundIndex(Context BackgroundContext, StringRef ResourceDir,
- const FileSystemProvider &,
- ArrayRef<std::string> URISchemes = {});
+ const FileSystemProvider &, ArrayRef<std::string> URISchemes);
~BackgroundIndex(); // Blocks while the current task finishes.
// Enqueue a translation unit for indexing.
@@ -52,18 +52,27 @@
// Wait until the queue is empty, to allow deterministic testing.
void blockUntilIdleForTest();
+ using FileDigest = decltype(llvm::SHA1::hash({}));
+ using FileDigests = llvm::StringMap<FileDigest>;
+
private:
+ void update(llvm::StringRef MainFile,
+ const llvm::StringMap<FileDigest> &FilesToUpdate,
+ SymbolSlab Symbols, RefSlab Refs);
+ llvm::StringRef uriToPath(llvm::StringRef FileURI, llvm::StringRef HintPath);
+
// configuration
std::string ResourceDir;
const FileSystemProvider &FSProvider;
Context BackgroundContext;
std::vector<std::string> URISchemes;
// index state
llvm::Error index(tooling::CompileCommand);
- FileSymbols IndexedSymbols; // Index contents.
- using Hash = decltype(llvm::SHA1::hash({}));
- llvm::StringMap<Hash> FileHash; // Digest of indexed file.
+
+ FileSymbols IndexedSymbols;
+ FileDigests IndexedFileDigests; // Key is absolute file path.
+ llvm::StringMap<std::string> URIToPathCache;
// queue management
using Task = std::function<void()>; // FIXME: use multiple worker threads.
Index: clangd/index/Background.cpp
===================================================================
--- clangd/index/Background.cpp
+++ clangd/index/Background.cpp
@@ -12,9 +12,16 @@
#include "Compiler.h"
#include "Logger.h"
#include "Trace.h"
+#include "URI.h"
#include "index/IndexAction.h"
#include "index/MemIndex.h"
#include "index/Serialization.h"
+#include "index/SymbolCollector.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/SourceManager.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
#include "llvm/Support/SHA1.h"
#include <random>
@@ -112,6 +119,126 @@
std::move(Cmd)));
}
+inline BackgroundIndex::FileDigest digest(StringRef Content) {
+ return SHA1::hash({(const uint8_t *)Content.data(), Content.size()});
+}
+
+static Optional<BackgroundIndex::FileDigest> digestFile(const SourceManager &SM,
+ FileID FID) {
+ bool Invalid = false;
+ StringRef Content = SM.getBufferData(FID, &Invalid);
+ if (Invalid)
+ return None;
+ return digest(Content);
+}
+
+StringRef BackgroundIndex::uriToPath(StringRef FileURI, StringRef HintPath) {
+ auto I = URIToPathCache.try_emplace(FileURI);
+ if (I.second) {
+ auto U = URI::parse(FileURI);
+ if (!U) {
+ elog("Failed to parse URI {0}: {1}", FileURI, U.takeError());
+ assert(false && "Failed to parse URI");
+ return "";
+ }
+ auto Path = URI::resolve(*U, HintPath);
+ if (!Path) {
+ elog("Failed to resolve URI {0}: {1}", FileURI, Path.takeError());
+ assert(false && "Failed to resolve URI");
+ I.first->second = *Path;
+ return "";
+ }
+ I.first->second = *Path;
+ }
+ return I.first->second;
+}
+
+// Given index results from an index action, update symbols/references for files
+// that have changed.
+void BackgroundIndex::update(StringRef MainFile,
+ const StringMap<FileDigest> &FilesToUpdate,
+ SymbolSlab Symbols, RefSlab Refs) {
+ // Partition symbols/references into files.
+ struct File {
+ DenseSet<const Symbol *> Symbols;
+ DenseSet<const Ref *> Refs;
+ };
+ StringMap<File> Files;
+ for (const auto &Sym : Symbols) {
+ if (Sym.CanonicalDeclaration) {
+ auto DeclPath = uriToPath(Sym.CanonicalDeclaration.FileURI, MainFile);
+ if (FilesToUpdate.count(DeclPath) != 0)
+ Files[DeclPath].Symbols.insert(&Sym);
+ }
+ // For symbols with different declaration and definition locations, we store
+ // the full symbol in both the header file and the implementation file, so
+ // that merging can tell the preferred symbols (from canonical headers) from
+ // other symbols (e.g. forward declarations).
+ if (Sym.Definition &&
+ Sym.Definition.FileURI != Sym.CanonicalDeclaration.FileURI) {
+ auto DefPath = uriToPath(Sym.Definition.FileURI, MainFile);
+ if (FilesToUpdate.count(DefPath) != 0)
+ Files[DefPath].Symbols.insert(&Sym);
+ }
+ }
+ DenseMap<const Ref *, SymbolID> RefToIDs;
+ for (const auto &SymRefs : Refs) {
+ for (const auto &R : SymRefs.second) {
+ auto Path = uriToPath(R.Location.FileURI, MainFile);
+ if (FilesToUpdate.count(Path) != 0) {
+ auto &F = Files[Path];
+ RefToIDs[&R] = SymRefs.first;
+ F.Refs.insert(&R);
+ }
+ }
+ }
+
+ // Build and store new slabs for each updated file.
+ for (const auto &F : Files) {
+ StringRef Path = F.first();
+ vlog("Update symbols in {0}", Path);
+ SymbolSlab::Builder Syms;
+ RefSlab::Builder Refs;
+ for (const auto *S : F.second.Symbols)
+ Syms.insert(*S);
+ for (const auto *R : F.second.Refs)
+ Refs.insert(RefToIDs[R], *R);
+ IndexedFileDigests[Path] = FilesToUpdate.lookup(Path);
+ IndexedSymbols.update(Path,
+ make_unique<SymbolSlab>(std::move(Syms).build()),
+ make_unique<RefSlab>(std::move(Refs).build()));
+ }
+}
+
+decltype(SymbolCollector::Options::FileFilter)
+createFileFilter(const BackgroundIndex::FileDigests &IndexedFileDigests,
+ StringMap<BackgroundIndex::FileDigest> &FilesToUpdate) {
+ return [&IndexedFileDigests, &FilesToUpdate](const SourceManager &SM,
+ FileID FID) {
+ StringRef Path;
+ if (const auto *F = SM.getFileEntryForID(FID))
+ Path = F->getName();
+ if (Path.empty())
+ return false; // Skip invalid files.
+ SmallString<128> AbsPath(Path);
+ if (std::error_code EC =
+ SM.getFileManager().getVirtualFileSystem()->makeAbsolute(AbsPath)) {
+ elog("Warning: could not make absolute file: {0}", EC.message());
+ return false; // Skip files without absolute path.
+ }
+ sys::path::remove_dots(AbsPath, /*remove_dot_dot=*/true);
+ auto Digest = digestFile(SM, FID);
+ if (!Digest)
+ return false;
+ auto D = IndexedFileDigests.find(AbsPath);
+ if (D != IndexedFileDigests.end() && D->second == Digest)
+ return false; // Skip files that haven't changed.
+
+ FilesToUpdate[AbsPath] = *Digest;
+ return true;
+ };
+}
+
Error BackgroundIndex::index(tooling::CompileCommand Cmd) {
trace::Span Tracer("BackgroundIndex");
SPAN_ATTACH(Tracer, "file", Cmd.Filename);
@@ -127,10 +254,9 @@
auto Buf = FS->getBufferForFile(AbsolutePath);
if (!Buf)
return errorCodeToError(Buf.getError());
- StringRef Contents = Buf->get()->getBuffer();
- auto Hash = SHA1::hash({(const uint8_t *)Contents.data(), Contents.size()});
+ auto Hash = digest(Buf->get()->getBuffer());
- if (FileHash.lookup(AbsolutePath) == Hash) {
+ if (IndexedFileDigests.lookup(AbsolutePath) == Hash) {
vlog("No need to index {0}, already up to date", AbsolutePath);
return Error::success();
}
@@ -153,9 +279,11 @@
"Couldn't build compiler instance");
SymbolCollector::Options IndexOpts;
+ IndexOpts.URISchemes = URISchemes;
+ StringMap<FileDigest> FilesToUpdate;
+ IndexOpts.FileFilter = createFileFilter(IndexedFileDigests, FilesToUpdate);
SymbolSlab Symbols;
RefSlab Refs;
- IndexFileIn IndexData;
auto Action = createStaticIndexingAction(
IndexOpts, [&](SymbolSlab S) { Symbols = std::move(S); },
[&](RefSlab R) { Refs = std::move(R); });
@@ -177,16 +305,15 @@
Symbols.size(), Refs.numRefs());
SPAN_ATTACH(Tracer, "symbols", int(Symbols.size()));
SPAN_ATTACH(Tracer, "refs", int(Refs.numRefs()));
- // FIXME: partition the symbols by file rather than TU, to avoid duplication.
- IndexedSymbols.update(AbsolutePath,
- llvm::make_unique<SymbolSlab>(std::move(Symbols)),
- llvm::make_unique<RefSlab>(std::move(Refs)));
- FileHash[AbsolutePath] = Hash;
+ update(AbsolutePath, FilesToUpdate, std::move(Symbols), std::move(Refs));
+ IndexedFileDigests[AbsolutePath] = Hash;
// FIXME: this should rebuild once-in-a-while, not after every file.
// At that point we should use Dex, too.
vlog("Rebuilding automatic index");
- reset(IndexedSymbols.buildIndex(IndexType::Light, URISchemes));
+ reset(IndexedSymbols.buildIndex(IndexType::Light, DuplicateHandling::Merge,
+ URISchemes));
+
return Error::success();
}
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
