hokein created this revision.
hokein added a reviewer: sammccall.
Herald added subscribers: kadircet, arphaman, jkorous, MaskRay, ioeric,
ilya-biryukov.
Implement the interface in
- FileIndex
- MemIndex
- MergeIndex
Depends on https://reviews.llvm.org/D50385.
Repository:
rCTE Clang Tools Extra
https://reviews.llvm.org/D51279
Files:
clangd/index/FileIndex.cpp
clangd/index/FileIndex.h
clangd/index/Index.h
clangd/index/MemIndex.cpp
clangd/index/MemIndex.h
clangd/index/Merge.cpp
unittests/clangd/FileIndexTests.cpp
unittests/clangd/IndexTests.cpp
unittests/clangd/TestTU.cpp
unittests/clangd/TestTU.h
Index: unittests/clangd/TestTU.h
===================================================================
--- unittests/clangd/TestTU.h
+++ unittests/clangd/TestTU.h
@@ -38,6 +38,16 @@
return TU;
}
+ static TestTU withAllCode(llvm::StringRef HeaderCode, llvm::StringRef Code,
+ llvm::StringRef Filename = "") {
+ TestTU TU;
+ TU.HeaderCode = HeaderCode;
+ TU.Code = Code;
+ if (!Filename.empty())
+ TU.Filename = Filename;
+ return TU;
+ }
+
// The code to be compiled.
std::string Code;
std::string Filename = "TestTU.cpp";
Index: unittests/clangd/TestTU.cpp
===================================================================
--- unittests/clangd/TestTU.cpp
+++ unittests/clangd/TestTU.cpp
@@ -45,7 +45,7 @@
SymbolSlab TestTU::headerSymbols() const {
auto AST = build();
- return indexAST(AST.getASTContext(), AST.getPreprocessorPtr());
+ return indexAST(AST.getASTContext(), AST.getPreprocessorPtr()).first;
}
std::unique_ptr<SymbolIndex> TestTU::index() const {
Index: unittests/clangd/IndexTests.cpp
===================================================================
--- unittests/clangd/IndexTests.cpp
+++ unittests/clangd/IndexTests.cpp
@@ -8,20 +8,33 @@
//===----------------------------------------------------------------------===//
#include "TestIndex.h"
+#include "TestTU.h"
+#include "Annotations.h"
#include "index/Index.h"
#include "index/MemIndex.h"
#include "index/Merge.h"
+#include "index/FileIndex.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
using testing::Pointee;
using testing::UnorderedElementsAre;
+using testing::AllOf;
namespace clang {
namespace clangd {
namespace {
MATCHER_P(Named, N, "") { return arg.Name == N; }
+MATCHER_P(OccurrenceRange, Range, "") {
+ return std::tie(arg.Location.Start.Line,
+ arg.Location.Start.Column,
+ arg.Location.End.Line,
+ arg.Location.End.Column) ==
+ std::tie(Range.start.line, Range.start.character, Range.end.line,
+ Range.end.character);
+}
+MATCHER_P(FileURI, F, "") { return arg.Location.FileURI == F; }
TEST(SymbolSlab, FindAndIterate) {
SymbolSlab::Builder B;
@@ -243,6 +256,55 @@
EXPECT_EQ(M.Name, "right");
}
+TEST(MergeIndexTest, FindOccurrences) {
+ FileIndex Dyn({"unittest"});
+ FileIndex StaticIndex({"unittest"});
+ auto MergedIndex = mergeIndex(&Dyn, &StaticIndex);
+
+ const char* HeaderCode = "class Foo;";
+ auto HeaderSymbols = TestTU::withHeaderCode("class Foo;").headerSymbols();
+ auto Foo = findSymbol(HeaderSymbols, "Foo");
+
+ // Build dynamic index.
+ Annotations Test1Code(R"(class $Foo[[Foo]];)");
+ auto AST =
+ TestTU::withAllCode(HeaderCode, Test1Code.code(), "test.cc").build();
+ Dyn.update("test.cc", &AST.getASTContext(), AST.getPreprocessorPtr(),
+ AST.getLocalTopLevelDecls());
+
+ // Build static index.
+ auto StaticAST =
+ TestTU::withAllCode(HeaderCode, "// static\nclass Foo {};", "test.cc")
+ .build();
+ // Add stale occurrences for test.cc.
+ StaticIndex.update("test.cc", &StaticAST.getASTContext(),
+ StaticAST.getPreprocessorPtr(),
+ StaticAST.getLocalTopLevelDecls());
+
+ // Add occcurrences for test2.cc
+ Annotations Test2Code(R"(class $Foo[[Foo]] {};)");
+ StaticAST =
+ TestTU::withAllCode(HeaderCode, Test2Code.code(), "test2.cc").build();
+ StaticIndex.update("test2.cc", &StaticAST.getASTContext(),
+ StaticAST.getPreprocessorPtr(),
+ StaticAST.getLocalTopLevelDecls());
+
+ OccurrencesRequest Request;
+ Request.IDs = {Foo.ID};
+ Request.Filter = SymbolOccurrenceKind::Declaration |
+ SymbolOccurrenceKind::Definition |
+ SymbolOccurrenceKind::Reference;
+ std::vector<SymbolOccurrence> Results;
+ MergedIndex->findOccurrences(
+ Request, [&](const SymbolOccurrence &O) { Results.push_back(O); });
+
+ EXPECT_THAT(Results,
+ UnorderedElementsAre(AllOf(OccurrenceRange(Test1Code.range("Foo")),
+ FileURI("unittest:///test.cc")),
+ AllOf(OccurrenceRange(Test2Code.range("Foo")),
+ FileURI("unittest:///test2.cc"))));
+}
+
} // namespace
} // namespace clangd
} // namespace clang
Index: unittests/clangd/FileIndexTests.cpp
===================================================================
--- unittests/clangd/FileIndexTests.cpp
+++ unittests/clangd/FileIndexTests.cpp
@@ -7,6 +7,7 @@
//
//===----------------------------------------------------------------------===//
+#include "Annotations.h"
#include "ClangdUnit.h"
#include "TestFS.h"
#include "TestTU.h"
@@ -19,6 +20,19 @@
#include "gtest/gtest.h"
using testing::UnorderedElementsAre;
+using testing::AllOf;
+
+MATCHER_P(OccurrenceRange, Range, "") {
+ return std::tie(arg.Location.Start.Line,
+ arg.Location.Start.Column,
+ arg.Location.End.Line,
+ arg.Location.End.Column) ==
+ std::tie(Range.start.line, Range.start.character, Range.end.line,
+ Range.end.character);
+}
+MATCHER_P(FileURI, F, "") {
+ return arg.Location.FileURI == F;
+}
namespace clang {
namespace clangd {
@@ -270,6 +284,46 @@
UnorderedElementsAre("ns_in_header", "ns_in_header::func_in_header"));
}
+TEST(FileIndexTest, Occurrences) {
+ const char* HeaderCode = "class Foo {};";
+ Annotations MainCode(R"cpp(
+ void f() {
+ $foo[[Foo]] foo;
+ }
+ )cpp");
+
+ auto Foo =
+ findSymbol(TestTU::withHeaderCode(HeaderCode).headerSymbols(), "Foo");
+
+ OccurrencesRequest Request;
+ Request.IDs = {Foo.ID};
+ Request.Filter = SymbolOccurrenceKind::Declaration |
+ SymbolOccurrenceKind::Definition |
+ SymbolOccurrenceKind::Reference;
+
+ FileIndex Index(/*URISchemes*/{"unittest"});
+ // Add test.cc
+ auto AST =
+ TestTU::withAllCode(HeaderCode, MainCode.code(), "test.cc").build();
+ Index.update("test.cc", &AST.getASTContext(), AST.getPreprocessorPtr(),
+ AST.getLocalTopLevelDecls());
+ // Add test2.cc
+ AST = TestTU::withAllCode(HeaderCode, MainCode.code(), "test2.cc").build();
+ Index.update("test2.cc", &AST.getASTContext(), AST.getPreprocessorPtr(),
+ AST.getLocalTopLevelDecls());
+
+ std::vector<SymbolOccurrence> Results;
+ Index.findOccurrences(Request, [&Results](const SymbolOccurrence& O) {
+ Results.push_back(O);
+ });
+
+ EXPECT_THAT(Results,
+ UnorderedElementsAre(AllOf(OccurrenceRange(MainCode.range("foo")),
+ FileURI("unittest:///test.cc")),
+ AllOf(OccurrenceRange(MainCode.range("foo")),
+ FileURI("unittest:///test2.cc"))));
+}
+
} // namespace
} // namespace clangd
} // namespace clang
Index: clangd/index/Merge.cpp
===================================================================
--- clangd/index/Merge.cpp
+++ clangd/index/Merge.cpp
@@ -81,7 +81,21 @@
void findOccurrences(const OccurrencesRequest &Req,
llvm::function_ref<void(const SymbolOccurrence &)>
Callback) const override {
- log("findOccurrences is not implemented.");
+ // Store all seen files by dynamic index.
+ llvm::DenseSet<llvm::StringRef> SeenFiles;
+ // Emit all dynamic occurrences, and static occurrences that are not
+ // in seen files.
+ // FIXME: If a file has been updated, and there are no occurrences indexed
+ // in dynamic index, stale results are still returned (from static index).
+ Dynamic->findOccurrences(Req, [&](const SymbolOccurrence &O) {
+ SeenFiles.insert(O.Location.FileURI);
+ Callback(O);
+ });
+ Static->findOccurrences(Req, [&](const SymbolOccurrence& O) {
+ if (llvm::is_contained(SeenFiles, O.Location.FileURI))
+ return;
+ Callback(O);
+ });
}
private:
Index: clangd/index/MemIndex.h
===================================================================
--- clangd/index/MemIndex.h
+++ clangd/index/MemIndex.h
@@ -22,7 +22,9 @@
public:
/// \brief (Re-)Build index for `Symbols`. All symbol pointers must remain
/// accessible as long as `Symbols` is kept alive.
- void build(std::shared_ptr<std::vector<const Symbol *>> Symbols);
+ void
+ build(std::shared_ptr<std::vector<const Symbol *>> Symbols,
+ std::vector<std::shared_ptr<SymbolOccurrenceSlab>> OccurrenceSlabs = {});
/// \brief Build index from a symbol slab.
static std::unique_ptr<SymbolIndex> build(SymbolSlab Slab);
@@ -44,6 +46,8 @@
// Index is a set of symbols that are deduplicated by symbol IDs.
// FIXME: build smarter index structure.
llvm::DenseMap<SymbolID, const Symbol *> Index;
+
+ std::vector<std::shared_ptr<SymbolOccurrenceSlab>> OccurrenceSlabs;
mutable std::mutex Mutex;
};
Index: clangd/index/MemIndex.cpp
===================================================================
--- clangd/index/MemIndex.cpp
+++ clangd/index/MemIndex.cpp
@@ -15,7 +15,9 @@
namespace clang {
namespace clangd {
-void MemIndex::build(std::shared_ptr<std::vector<const Symbol *>> Syms) {
+void MemIndex::build(
+ std::shared_ptr<std::vector<const Symbol *>> Syms,
+ std::vector<std::shared_ptr<SymbolOccurrenceSlab>> Occurrences) {
llvm::DenseMap<SymbolID, const Symbol *> TempIndex;
for (const Symbol *Sym : *Syms)
TempIndex[Sym->ID] = Sym;
@@ -25,12 +27,13 @@
std::lock_guard<std::mutex> Lock(Mutex);
Index = std::move(TempIndex);
Symbols = std::move(Syms); // Relase old symbols.
+ OccurrenceSlabs = std::move(Occurrences);
}
}
std::unique_ptr<SymbolIndex> MemIndex::build(SymbolSlab Slab) {
auto Idx = llvm::make_unique<MemIndex>();
- Idx->build(getSymbolsFromSlab(std::move(Slab)));
+ Idx->build(getSymbolsFromSlab(std::move(Slab)), {});
return std::move(Idx);
}
@@ -81,7 +84,15 @@
void MemIndex::findOccurrences(
const OccurrencesRequest &Req,
llvm::function_ref<void(const SymbolOccurrence &)> Callback) const {
- log("findOccurrences is not implemented.");
+ for (const auto &Slab : OccurrenceSlabs) {
+ for (const auto &ID : Req.IDs) {
+ for (const auto &Occurrence : Slab->find(ID)) {
+ if (static_cast<int>(Req.Filter & Occurrence.Kind)) {
+ Callback(Occurrence);
+ }
+ }
+ }
+ }
}
std::shared_ptr<std::vector<const Symbol *>>
Index: clangd/index/Index.h
===================================================================
--- clangd/index/Index.h
+++ clangd/index/Index.h
@@ -369,6 +369,15 @@
const_iterator begin() const { return Occurrences.begin(); }
const_iterator end() const { return Occurrences.end(); }
+ size_t bytes() const {
+ return sizeof(*this) + Arena.getTotalMemory() +
+ Occurrences.getMemorySize();
+ }
+
+ size_t size() const {
+ return Occurrences.size();
+ }
+
// Adds a symbol occurrence.
// This is a deep copy: underlying FileURI will be owned by the slab.
void insert(const SymbolID &SymID, const SymbolOccurrence &Occurrence);
Index: clangd/index/FileIndex.h
===================================================================
--- clangd/index/FileIndex.h
+++ clangd/index/FileIndex.h
@@ -41,16 +41,21 @@
public:
/// \brief Updates all symbols in a file. If \p Slab is nullptr, symbols for
/// \p Path will be removed.
- void update(PathRef Path, std::unique_ptr<SymbolSlab> Slab);
+ void update(PathRef Path, std::unique_ptr<SymbolSlab> Slab,
+ std::unique_ptr<SymbolOccurrenceSlab> Occurrences = nullptr);
// The shared_ptr keeps the symbols alive
std::shared_ptr<std::vector<const Symbol *>> allSymbols();
+ std::vector<std::shared_ptr<SymbolOccurrenceSlab>> allOccurrenceSlabs() const;
+
private:
mutable std::mutex Mutex;
/// \brief Stores the latest snapshots for all active files.
llvm::StringMap<std::shared_ptr<SymbolSlab>> FileToSlabs;
+ /// \breif Stores the latest occurrence slabs for all active files.
+ llvm::StringMap<std::shared_ptr<SymbolOccurrenceSlab>> FileToOccurrenceSlabs;
};
/// \brief This manages symbls from files and an in-memory index on all symbols.
@@ -92,7 +97,7 @@
/// If URISchemes is empty, the default schemes in SymbolCollector will be used.
/// If \p TopLevelDecls is set, only these decls are indexed. Otherwise, all top
/// level decls obtained from \p AST are indexed.
-SymbolSlab
+std::pair<SymbolSlab, SymbolOccurrenceSlab>
indexAST(ASTContext &AST, std::shared_ptr<Preprocessor> PP,
llvm::Optional<llvm::ArrayRef<Decl *>> TopLevelDecls = llvm::None,
llvm::ArrayRef<std::string> URISchemes = {});
Index: clangd/index/FileIndex.cpp
===================================================================
--- clangd/index/FileIndex.cpp
+++ clangd/index/FileIndex.cpp
@@ -16,9 +16,10 @@
namespace clang {
namespace clangd {
-SymbolSlab indexAST(ASTContext &AST, std::shared_ptr<Preprocessor> PP,
- llvm::Optional<llvm::ArrayRef<Decl *>> TopLevelDecls,
- llvm::ArrayRef<std::string> URISchemes) {
+std::pair<SymbolSlab, SymbolOccurrenceSlab>
+indexAST(ASTContext &AST, std::shared_ptr<Preprocessor> PP,
+ llvm::Optional<llvm::ArrayRef<Decl *>> TopLevelDecls,
+ llvm::ArrayRef<std::string> URISchemes) {
SymbolCollector::Options CollectorOpts;
// FIXME(ioeric): we might also want to collect include headers. We would need
// to make sure all includes are canonicalized (with CanonicalIncludes), which
@@ -31,8 +32,6 @@
CollectorOpts.URISchemes = URISchemes;
CollectorOpts.Origin = SymbolOrigin::Dynamic;
- SymbolCollector Collector(std::move(CollectorOpts));
- Collector.setPreprocessor(PP);
index::IndexingOptions IndexOpts;
// We only need declarations, because we don't count references.
IndexOpts.SystemSymbolFilter =
@@ -46,20 +45,41 @@
DeclsToIndex.assign(AST.getTranslationUnitDecl()->decls().begin(),
AST.getTranslationUnitDecl()->decls().end());
+ if (TopLevelDecls) { // index main AST, set occurrence flag.
+ CollectorOpts.OccurrenceFilter = SymbolOccurrenceKind::Declaration |
+ SymbolOccurrenceKind::Definition |
+ SymbolOccurrenceKind::Reference;
+ }
+
+ SymbolCollector Collector(std::move(CollectorOpts));
+ Collector.setPreprocessor(PP);
index::indexTopLevelDecls(AST, DeclsToIndex, Collector, IndexOpts);
- return Collector.takeSymbols();
+ vlog("index for AST: ");
+ auto Syms = Collector.takeSymbols();
+ auto Occurrences = Collector.takeOccurrences();
+ vlog(" symbol slab: {0} symbols, {1} bytes", Syms.size(), Syms.bytes());
+ vlog(" occurrence slab: {0} symbols, {1} bytes", Occurrences.size(),
+ Occurrences.bytes());
+ return {std::move(Syms), std::move(Occurrences)};
}
FileIndex::FileIndex(std::vector<std::string> URISchemes)
: URISchemes(std::move(URISchemes)) {}
-void FileSymbols::update(PathRef Path, std::unique_ptr<SymbolSlab> Slab) {
+void FileSymbols::update(PathRef Path, std::unique_ptr<SymbolSlab> Slab,
+ std::unique_ptr<SymbolOccurrenceSlab> Occurrences) {
std::lock_guard<std::mutex> Lock(Mutex);
- if (!Slab)
+ if (!Slab) {
FileToSlabs.erase(Path);
- else
+ } else {
FileToSlabs[Path] = std::move(Slab);
+ }
+ if (!Occurrences) {
+ FileToOccurrenceSlabs.erase(Path);
+ } else {
+ FileToOccurrenceSlabs[Path] = std::move(Occurrences);
+ }
}
std::shared_ptr<std::vector<const Symbol *>> FileSymbols::allSymbols() {
@@ -85,19 +105,32 @@
return {std::move(Snap), Pointers};
}
+std::vector<std::shared_ptr<SymbolOccurrenceSlab>>
+FileSymbols::allOccurrenceSlabs() const{
+ std::vector<std::shared_ptr<SymbolOccurrenceSlab>> Slabs;
+ std::lock_guard<std::mutex> Lock(Mutex);
+
+ for (const auto &FileAndSlab : FileToOccurrenceSlabs)
+ Slabs.push_back(FileAndSlab.second);
+ return Slabs;
+}
+
void FileIndex::update(PathRef Path, ASTContext *AST,
std::shared_ptr<Preprocessor> PP,
llvm::Optional<llvm::ArrayRef<Decl *>> TopLevelDecls) {
if (!AST) {
- FSymbols.update(Path, nullptr);
+ FSymbols.update(Path, nullptr, nullptr);
} else {
assert(PP);
auto Slab = llvm::make_unique<SymbolSlab>();
- *Slab = indexAST(*AST, PP, TopLevelDecls, URISchemes);
- FSymbols.update(Path, std::move(Slab));
+ auto IndexResults = indexAST(*AST, PP, TopLevelDecls, URISchemes);
+ *Slab = std::move(IndexResults.first);
+ auto OccurrenceSlab = llvm::make_unique<SymbolOccurrenceSlab>();
+ *OccurrenceSlab = std::move(IndexResults.second);
+ FSymbols.update(Path, std::move(Slab), std::move(OccurrenceSlab));
}
auto Symbols = FSymbols.allSymbols();
- Index.build(std::move(Symbols));
+ Index.build(std::move(Symbols), FSymbols.allOccurrenceSlabs());
}
bool FileIndex::fuzzyFind(
@@ -115,7 +148,7 @@
void FileIndex::findOccurrences(
const OccurrencesRequest &Req,
llvm::function_ref<void(const SymbolOccurrence &)> Callback) const {
- log("findOccurrences is not implemented.");
+ Index.findOccurrences(Req, Callback);
}
} // namespace clangd
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
