Author: sammccall Date: Thu Oct 4 02:16:12 2018 New Revision: 343760 URL: http://llvm.org/viewvc/llvm-project?rev=343760&view=rev Log: [clangd] Support refs() in dex. Largely cloned from MemIndex.
Reviewers: hokein Subscribers: ilya-biryukov, ioeric, MaskRay, jkorous, arphaman, kadircet, cfe-commits Differential Revision: https://reviews.llvm.org/D52726 Modified: clang-tools-extra/trunk/clangd/index/Serialization.cpp clang-tools-extra/trunk/clangd/index/dex/Dex.cpp clang-tools-extra/trunk/clangd/index/dex/Dex.h clang-tools-extra/trunk/clangd/indexer/IndexerMain.cpp clang-tools-extra/trunk/unittests/clangd/DexTests.cpp Modified: clang-tools-extra/trunk/clangd/index/Serialization.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Serialization.cpp?rev=343760&r1=343759&r2=343760&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/index/Serialization.cpp (original) +++ clang-tools-extra/trunk/clangd/index/Serialization.cpp Thu Oct 4 02:16:12 2018 @@ -435,8 +435,9 @@ std::unique_ptr<SymbolIndex> loadIndex(l } trace::Span Tracer("BuildIndex"); - auto Index = UseDex ? dex::Dex::build(std::move(Symbols), URISchemes) - : MemIndex::build(std::move(Symbols), std::move(Refs)); + auto Index = + UseDex ? dex::Dex::build(std::move(Symbols), std::move(Refs), URISchemes) + : MemIndex::build(std::move(Symbols), std::move(Refs)); vlog("Loaded {0} from {1} with estimated memory usage {2}", UseDex ? "Dex" : "MemIndex", SymbolFilename, Index->estimateMemoryUsage()); Modified: clang-tools-extra/trunk/clangd/index/dex/Dex.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Dex.cpp?rev=343760&r1=343759&r2=343760&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/index/dex/Dex.cpp (original) +++ clang-tools-extra/trunk/clangd/index/dex/Dex.cpp Thu Oct 4 02:16:12 2018 @@ -24,6 +24,15 @@ namespace clang { namespace clangd { namespace dex { +std::unique_ptr<SymbolIndex> +Dex::build(SymbolSlab Symbols, RefSlab Refs, + llvm::ArrayRef<std::string> URISchemes) { + auto Size = Symbols.bytes() + Refs.bytes(); + auto Data = std::make_pair(std::move(Symbols), std::move(Refs)); + return llvm::make_unique<Dex>(Data.first, Data.second, std::move(Data), Size, + std::move(URISchemes)); +} + namespace { // Mark symbols which are can be used for code completion. @@ -254,7 +263,10 @@ void Dex::lookup(const LookupRequest &Re void Dex::refs(const RefsRequest &Req, llvm::function_ref<void(const Ref &)> Callback) const { trace::Span Tracer("Dex refs"); - log("refs is not implemented."); + for (const auto &ID : Req.IDs) + for (const auto &Ref : Refs.lookup(ID)) + if (static_cast<int>(Req.Filter & Ref.Kind)) + Callback(Ref); } size_t Dex::estimateMemoryUsage() const { @@ -264,6 +276,7 @@ size_t Dex::estimateMemoryUsage() const Bytes += InvertedIndex.getMemorySize(); for (const auto &TokenToPostingList : InvertedIndex) Bytes += TokenToPostingList.second.bytes(); + Bytes += Refs.getMemorySize(); return Bytes + BackingDataSize; } Modified: clang-tools-extra/trunk/clangd/index/dex/Dex.h URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Dex.h?rev=343760&r1=343759&r2=343760&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/index/dex/Dex.h (original) +++ clang-tools-extra/trunk/clangd/index/dex/Dex.h Thu Oct 4 02:16:12 2018 @@ -41,9 +41,10 @@ namespace dex { // index on disk and then load it if available. class Dex : public SymbolIndex { public: - // All symbols must outlive this index. - template <typename Range> - Dex(Range &&Symbols, llvm::ArrayRef<std::string> Schemes) + // All data must outlive this index. + template <typename SymbolRange, typename RefsRange> + Dex(SymbolRange &&Symbols, RefsRange &&Refs, + llvm::ArrayRef<std::string> Schemes) : Corpus(0), URISchemes(Schemes) { // If Schemes don't contain any items, fall back to SymbolCollector's // default URI schemes. @@ -53,26 +54,24 @@ public: } for (auto &&Sym : Symbols) this->Symbols.push_back(&Sym); + for (auto &&Ref : Refs) + this->Refs.try_emplace(Ref.first, Ref.second); buildIndex(); } - // Symbols are owned by BackingData, Index takes ownership. - template <typename Range, typename Payload> - Dex(Range &&Symbols, Payload &&BackingData, size_t BackingDataSize, - llvm::ArrayRef<std::string> URISchemes) - : Dex(std::forward<Range>(Symbols), URISchemes) { + // Symbols and Refs are owned by BackingData, Index takes ownership. + template <typename SymbolRange, typename RefsRange, typename Payload> + Dex(SymbolRange &&Symbols, RefsRange &&Refs, Payload &&BackingData, + size_t BackingDataSize, llvm::ArrayRef<std::string> URISchemes) + : Dex(std::forward<SymbolRange>(Symbols), std::forward<RefsRange>(Refs), + URISchemes) { KeepAlive = std::shared_ptr<void>( std::make_shared<Payload>(std::move(BackingData)), nullptr); this->BackingDataSize = BackingDataSize; } - /// Builds an index from a slab. The index takes ownership of the slab. + /// Builds an index from slabs. The index takes ownership of the slab. static std::unique_ptr<SymbolIndex> - build(SymbolSlab Slab, llvm::ArrayRef<std::string> URISchemes) { - // Store Slab size before it is moved. - const auto BackingDataSize = Slab.bytes(); - return llvm::make_unique<Dex>(Slab, std::move(Slab), BackingDataSize, - URISchemes); - } + build(SymbolSlab, RefSlab, llvm::ArrayRef<std::string> URISchemes); bool fuzzyFind(const FuzzyFindRequest &Req, @@ -102,6 +101,7 @@ private: /// during the fuzzyFind process. llvm::DenseMap<Token, PostingList> InvertedIndex; dex::Corpus Corpus; + llvm::DenseMap<SymbolID, llvm::ArrayRef<Ref>> Refs; std::shared_ptr<void> KeepAlive; // poor man's move-only std::any // Size of memory retained by KeepAlive. size_t BackingDataSize = 0; Modified: clang-tools-extra/trunk/clangd/indexer/IndexerMain.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/indexer/IndexerMain.cpp?rev=343760&r1=343759&r2=343760&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/indexer/IndexerMain.cpp (original) +++ clang-tools-extra/trunk/clangd/indexer/IndexerMain.cpp Thu Oct 4 02:16:12 2018 @@ -47,7 +47,7 @@ static llvm::cl::opt<IndexFileFormat> "human-readable YAML format"), clEnumValN(IndexFileFormat::RIFF, "binary", "binary RIFF format")), - llvm::cl::init(IndexFileFormat::YAML)); + llvm::cl::init(IndexFileFormat::RIFF)); class IndexActionFactory : public tooling::FrontendActionFactory { public: Modified: clang-tools-extra/trunk/unittests/clangd/DexTests.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/DexTests.cpp?rev=343760&r1=343759&r2=343760&view=diff ============================================================================== --- clang-tools-extra/trunk/unittests/clangd/DexTests.cpp (original) +++ clang-tools-extra/trunk/unittests/clangd/DexTests.cpp Thu Oct 4 02:16:12 2018 @@ -423,7 +423,8 @@ TEST(DexSearchTokens, SymbolPath) { //===----------------------------------------------------------------------===// TEST(Dex, Lookup) { - auto I = Dex::build(generateSymbols({"ns::abc", "ns::xyz"}), URISchemes); + auto I = Dex::build(generateSymbols({"ns::abc", "ns::xyz"}), RefSlab(), + URISchemes); EXPECT_THAT(lookup(*I, SymbolID("ns::abc")), UnorderedElementsAre("ns::abc")); EXPECT_THAT(lookup(*I, {SymbolID("ns::abc"), SymbolID("ns::xyz")}), UnorderedElementsAre("ns::abc", "ns::xyz")); @@ -436,7 +437,7 @@ TEST(Dex, FuzzyFind) { auto Index = Dex::build(generateSymbols({"ns::ABC", "ns::BCD", "::ABC", "ns::nested::ABC", "other::ABC", "other::A"}), - URISchemes); + RefSlab(), URISchemes); FuzzyFindRequest Req; Req.Query = "ABC"; Req.Scopes = {"ns::"}; @@ -466,13 +467,13 @@ TEST(DexTest, DexDeduplicate) { symbol("2") /* duplicate */}; FuzzyFindRequest Req; Req.Query = "2"; - Dex I(Symbols, URISchemes); + Dex I(Symbols, RefSlab(), URISchemes); EXPECT_FALSE(Req.Limit); EXPECT_THAT(match(I, Req), ElementsAre("2", "2")); } TEST(DexTest, DexLimitedNumMatches) { - auto I = Dex::build(generateNumSymbols(0, 100), URISchemes); + auto I = Dex::build(generateNumSymbols(0, 100), RefSlab(), URISchemes); FuzzyFindRequest Req; Req.Query = "5"; Req.Limit = 3; @@ -486,7 +487,7 @@ TEST(DexTest, DexLimitedNumMatches) { TEST(DexTest, FuzzyMatch) { auto I = Dex::build( generateSymbols({"LaughingOutLoud", "LionPopulation", "LittleOldLady"}), - URISchemes); + RefSlab(), URISchemes); FuzzyFindRequest Req; Req.Query = "lol"; Req.Limit = 2; @@ -495,14 +496,16 @@ TEST(DexTest, FuzzyMatch) { } TEST(DexTest, MatchQualifiedNamesWithoutSpecificScope) { - auto I = Dex::build(generateSymbols({"a::y1", "b::y2", "y3"}), URISchemes); + auto I = Dex::build(generateSymbols({"a::y1", "b::y2", "y3"}), RefSlab(), + URISchemes); FuzzyFindRequest Req; Req.Query = "y"; EXPECT_THAT(match(*I, Req), UnorderedElementsAre("a::y1", "b::y2", "y3")); } TEST(DexTest, MatchQualifiedNamesWithGlobalScope) { - auto I = Dex::build(generateSymbols({"a::y1", "b::y2", "y3"}), URISchemes); + auto I = Dex::build(generateSymbols({"a::y1", "b::y2", "y3"}), RefSlab(), + URISchemes); FuzzyFindRequest Req; Req.Query = "y"; Req.Scopes = {""}; @@ -510,8 +513,9 @@ TEST(DexTest, MatchQualifiedNamesWithGlo } TEST(DexTest, MatchQualifiedNamesWithOneScope) { - auto I = Dex::build( - generateSymbols({"a::y1", "a::y2", "a::x", "b::y2", "y3"}), URISchemes); + auto I = + Dex::build(generateSymbols({"a::y1", "a::y2", "a::x", "b::y2", "y3"}), + RefSlab(), URISchemes); FuzzyFindRequest Req; Req.Query = "y"; Req.Scopes = {"a::"}; @@ -520,7 +524,7 @@ TEST(DexTest, MatchQualifiedNamesWithOne TEST(DexTest, MatchQualifiedNamesWithMultipleScopes) { auto I = Dex::build( - generateSymbols({"a::y1", "a::y2", "a::x", "b::y3", "y3"}), URISchemes); + generateSymbols({"a::y1", "a::y2", "a::x", "b::y3", "y3"}), RefSlab(), URISchemes); FuzzyFindRequest Req; Req.Query = "y"; Req.Scopes = {"a::", "b::"}; @@ -528,7 +532,7 @@ TEST(DexTest, MatchQualifiedNamesWithMul } TEST(DexTest, NoMatchNestedScopes) { - auto I = Dex::build(generateSymbols({"a::y1", "a::b::y2"}), URISchemes); + auto I = Dex::build(generateSymbols({"a::y1", "a::b::y2"}), RefSlab(), URISchemes); FuzzyFindRequest Req; Req.Query = "y"; Req.Scopes = {"a::"}; @@ -537,7 +541,7 @@ TEST(DexTest, NoMatchNestedScopes) { TEST(DexTest, WildcardScope) { auto I = - Dex::build(generateSymbols({"a::y1", "a::b::y2", "c::y3"}), URISchemes); + Dex::build(generateSymbols({"a::y1", "a::b::y2", "c::y3"}), RefSlab(), URISchemes); FuzzyFindRequest Req; Req.Query = "y"; Req.Scopes = {"a::"}; @@ -547,7 +551,7 @@ TEST(DexTest, WildcardScope) { } TEST(DexTest, IgnoreCases) { - auto I = Dex::build(generateSymbols({"ns::ABC", "ns::abc"}), URISchemes); + auto I = Dex::build(generateSymbols({"ns::ABC", "ns::abc"}), RefSlab(), URISchemes); FuzzyFindRequest Req; Req.Query = "AB"; Req.Scopes = {"ns::"}; @@ -555,7 +559,7 @@ TEST(DexTest, IgnoreCases) { } TEST(DexTest, Lookup) { - auto I = Dex::build(generateSymbols({"ns::abc", "ns::xyz"}), URISchemes); + auto I = Dex::build(generateSymbols({"ns::abc", "ns::xyz"}), RefSlab(), URISchemes); EXPECT_THAT(lookup(*I, SymbolID("ns::abc")), UnorderedElementsAre("ns::abc")); EXPECT_THAT(lookup(*I, {SymbolID("ns::abc"), SymbolID("ns::xyz")}), UnorderedElementsAre("ns::abc", "ns::xyz")); @@ -570,7 +574,7 @@ TEST(DexTest, SymbolIndexOptionsFilter) CodeCompletionSymbol.Flags = Symbol::SymbolFlag::IndexedForCodeCompletion; NonCodeCompletionSymbol.Flags = Symbol::SymbolFlag::None; std::vector<Symbol> Symbols{CodeCompletionSymbol, NonCodeCompletionSymbol}; - Dex I(Symbols, URISchemes); + Dex I(Symbols, RefSlab(), URISchemes); FuzzyFindRequest Req; Req.RestrictForCodeCompletion = false; EXPECT_THAT(match(I, Req), ElementsAre("Completion", "NoCompletion")); @@ -585,7 +589,7 @@ TEST(DexTest, ProximityPathsBoosting) { CloseSymbol.CanonicalDeclaration.FileURI = "unittest:///a/b/c/d/e/f/file.h"; std::vector<Symbol> Symbols{CloseSymbol, RootSymbol}; - Dex I(Symbols, URISchemes); + Dex I(Symbols, RefSlab(), URISchemes); FuzzyFindRequest Req; Req.Query = "abc"; @@ -603,6 +607,31 @@ TEST(DexTest, ProximityPathsBoosting) { EXPECT_THAT(match(I, Req), ElementsAre("root::abc")); } +TEST(DexTests, Refs) { + DenseMap<SymbolID, std::vector<Ref>> Refs; + auto AddRef = [&](const Symbol& Sym, StringRef Filename, RefKind Kind) { + auto& SymbolRefs = Refs[Sym.ID]; + SymbolRefs.emplace_back(); + SymbolRefs.back().Kind = Kind; + SymbolRefs.back().Location.FileURI = Filename; + }; + auto Foo = symbol("foo"); + auto Bar = symbol("bar"); + AddRef(Foo, "foo.h", RefKind::Declaration); + AddRef(Foo, "reffoo.h", RefKind::Reference); + AddRef(Bar, "bar.h", RefKind::Declaration); + + std::vector<std::string> Files; + RefsRequest Req; + Req.IDs.insert(Foo.ID); + Req.Filter = RefKind::Declaration | RefKind::Definition; + Dex(std::vector<Symbol>{Foo, Bar}, Refs, {}).refs(Req, [&](const Ref &R) { + Files.push_back(R.Location.FileURI); + }); + + EXPECT_THAT(Files, ElementsAre("foo.h")); +} + } // namespace } // namespace dex } // namespace clangd _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits