kbobyrev updated this revision to Diff 164834.
kbobyrev marked 8 inline comments as done.
kbobyrev added a comment.
Address a round of comments; implement a dummy ad-hoc subcommand parser.
https://reviews.llvm.org/D51628
Files:
clang-tools-extra/clangd/CMakeLists.txt
clang-tools-extra/clangd/index/dex/dexplorer/CMakeLists.txt
clang-tools-extra/clangd/index/dex/dexplorer/DExplorer.cpp
Index: clang-tools-extra/clangd/index/dex/dexplorer/DExplorer.cpp
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/index/dex/dexplorer/DExplorer.cpp
@@ -0,0 +1,186 @@
+//===--- DExplorer.cpp - Dex Exploration tool -------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a simple interactive tool which can be used to manually
+// evaluate symbol search quality of Clangd index.
+//
+//===----------------------------------------------------------------------===//
+
+#include "../../../index/SymbolYAML.h"
+#include "../Dex.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/LineEditor/LineEditor.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Signals.h"
+#include "llvm/Support/YAMLTraits.h"
+#include <cstdlib>
+
+using clang::clangd::FuzzyFindRequest;
+using clang::clangd::loadIndex;
+using clang::clangd::Symbol;
+using clang::clangd::SymbolIndex;
+using llvm::StringRef;
+
+namespace {
+
+llvm::cl::opt<std::string>
+ SymbolCollection("symbol-collection-file",
+ llvm::cl::desc("Path to the file with symbol collection"),
+ llvm::cl::Positional, llvm::cl::Required);
+
+static const std::string Overview = R"(
+This is an **experimental** interactive tool to process user-provided search
+queries over given symbol collection obtained via global-symbol-builder. The
+tool can be used to evaluate search quality of existing index implementations
+and manually construct non-trivial test cases.
+
+Type use "help" request to get information about the details.
+)";
+
+template <typename TimeUnit> std::string unitToString() { return ""; }
+
+template <> std::string unitToString<std::chrono::milliseconds>() {
+ return "ms";
+}
+
+template <> std::string unitToString<std::chrono::seconds>() { return "s"; }
+
+template <class Function, class TimeUnit = std::chrono::milliseconds>
+void reportTime(StringRef Name, Function F) {
+ const auto TimerStart = std::chrono::high_resolution_clock::now();
+ F();
+ const auto TimerStop = std::chrono::high_resolution_clock::now();
+ llvm::outs()
+ << Name << " took "
+ << std::chrono::duration_cast<TimeUnit>(TimerStop - TimerStart).count()
+ << ' ' << unitToString<TimeUnit>() << ".\n";
+}
+
+void fuzzyFindRequest(const std::unique_ptr<SymbolIndex> &Index,
+ const FuzzyFindRequest &Request) {
+ std::vector<Symbol> Symbols;
+ Index->fuzzyFind(Request, [&](const Symbol &S) { Symbols.push_back(S); });
+ // FIXME(kbobyrev): Print symbol final scores to see the distribution.
+ llvm::outs() << "\nRetrieved Symbols:\n";
+ llvm::outs() << "Symbol Name\n";
+ for (const auto &Symbol : Symbols)
+ llvm::outs() << Symbol.Name << '\n';
+ llvm::outs() << '\n';
+}
+
+static const std::string HelpMessage = R"(
+DExplorer commands:
+
+> fuzzy-find Name
+
+Constructs fuzzy find request given unqualified symbol name and returns top 10
+symbols retrieved from index.
+
+> lookup-id SymbolID
+
+Retrieves symbol names given USR.
+
+> help
+
+Shows this message.
+
+Press Ctrl-D to exit.
+)";
+
+void helpRequest() { llvm::outs() << HelpMessage; }
+
+void lookupRequest(const std::unique_ptr<SymbolIndex> &Index,
+ clang::clangd::LookupRequest &Request) {
+ std::vector<Symbol> Symbols;
+ Index->lookup(Request, [&](const Symbol &S) { Symbols.push_back(S); });
+ // FIXME(kbobyrev): Print symbol final scores to see the distribution.
+ llvm::outs() << "\nRetrieved Symbols:\n";
+ llvm::outs() << "Rank. Symbol Name | Symbol ID\n";
+ for (size_t Rank = 0; Rank < Symbols.size(); ++Rank)
+ llvm::outs() << Rank << ". " << Symbols[Rank].Name << " | "
+ << Symbols[Rank].ID.str() << '\n';
+ llvm::outs() << '\n';
+}
+
+// FIXME(kbobyrev): Make this an actual REPL: probably use LLVM Command Line
+// library for parsing flags and arguments.
+// FIXME(kbobyrev): Ideas for commands:
+// * symbol lookup: print out symbol in YAML format given SymbolID
+// * find symbol references: print set of reference locations
+// * load/swap/reload index: this would make it possible to get rid of llvm::cl
+// usages in the tool driver and actually use llvm::cl library in the REPL.
+// * show posting list density histogram (our dump data somewhere so that user
+// could build one)
+// * show number of tokens of each kind
+// * print out tokens with the most dense posting lists
+// * print out tokens with least dense posting lists
+void dispatchRequest(const std::unique_ptr<SymbolIndex> &Index,
+ StringRef Request) {
+ llvm::SmallVector<StringRef, 2> Arguments;
+ Request.split(Arguments, ' ');
+ if (Arguments.empty()) {
+ llvm::outs() << "ERROR: Request can not be empty.\n";
+ helpRequest();
+ return;
+ }
+
+ if (Arguments.front() == "fuzzy-find") {
+ if (Arguments.size() != 2) {
+ llvm::outs() << "ERROR: fuzzy-find request must specify unqualified "
+ "symbol name.\n";
+ helpRequest();
+ return;
+ }
+ FuzzyFindRequest Req;
+ Req.MaxCandidateCount = 10;
+ Req.Query = Arguments.back();
+ reportTime("fuzzy-find request", [&]() { fuzzyFindRequest(Index, Req); });
+ } else if (Arguments.front() == "lookup") {
+ if (Arguments.size() != 2) {
+ llvm::outs() << "ERROR: lookup-id request must specify symbol ID .\n";
+ helpRequest();
+ return;
+ }
+ llvm::DenseSet<clang::clangd::SymbolID> IDs{
+ clang::clangd::SymbolID{Arguments.back()}};
+ clang::clangd::LookupRequest Req{IDs};
+ lookupRequest(Index, Req);
+ } else {
+ helpRequest();
+ }
+}
+
+} // namespace
+
+int main(int argc, const char *argv[]) {
+ llvm::cl::ParseCommandLineOptions(argc, argv, Overview);
+ llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
+
+ // FIXME(kbobyrev): Wrap time measurements into something like
+ // measureTime(Function, Arguments...).
+ std::unique_ptr<SymbolIndex> Index;
+ reportTime<std::function<void()>, std::chrono::seconds>("Dex build", [&]() {
+ Index = loadIndex(SymbolCollection, /*URISchemes=*/{},
+ /*UseDex=*/true);
+ });
+
+ if (!Index) {
+ llvm::outs() << "ERROR: Please provide a valid YAML symbol collection.\n";
+ return -1;
+ }
+
+ llvm::LineEditor LE("dexplorer");
+
+ while (llvm::Optional<std::string> Line = LE.readLine())
+ dispatchRequest(Index, Line.getValue());
+
+ return 0;
+}
Index: clang-tools-extra/clangd/index/dex/dexplorer/CMakeLists.txt
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/index/dex/dexplorer/CMakeLists.txt
@@ -0,0 +1,15 @@
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../../)
+
+set(LLVM_LINK_COMPONENTS
+ LineEditor
+ Support
+ )
+
+add_clang_executable(dexplorer
+ DExplorer.cpp
+ )
+
+target_link_libraries(dexplorer
+ PRIVATE
+ clangDaemon
+)
Index: clang-tools-extra/clangd/CMakeLists.txt
===================================================================
--- clang-tools-extra/clangd/CMakeLists.txt
+++ clang-tools-extra/clangd/CMakeLists.txt
@@ -74,3 +74,4 @@
endif()
add_subdirectory(tool)
add_subdirectory(global-symbol-builder)
+add_subdirectory(index/dex/dexplorer)
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
