kbobyrev created this revision. kbobyrev added a reviewer: sammccall. Herald added subscribers: cfe-commits, usaxena95, kadircet, arphaman, jkorous, MaskRay, ilya-biryukov, mgorny. Herald added a project: clang.
- Make it possible for dexp to use remote server - Remove clangd-index-client (since it's now merged into dexp) - Implement `clangd::remote::IndexClient` that is derived from `SymbolIndex` - Slightly improve CMake infrastructure This is still WIP due to a couple of FIXMEs I need to resolve in this patch. Repository: rG LLVM Github Monorepo https://reviews.llvm.org/D78521 Files: clang-tools-extra/clangd/CMakeLists.txt clang-tools-extra/clangd/index/Serialization.h clang-tools-extra/clangd/index/YAMLSerialization.cpp clang-tools-extra/clangd/index/dex/dexp/CMakeLists.txt clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp clang-tools-extra/clangd/index/remote/CMakeLists.txt clang-tools-extra/clangd/index/remote/Index.cpp clang-tools-extra/clangd/index/remote/Index.h clang-tools-extra/clangd/index/remote/Index.proto clang-tools-extra/clangd/index/remote/client/CMakeLists.txt clang-tools-extra/clangd/index/remote/client/Client.cpp clang-tools-extra/clangd/index/remote/server/Server.cpp llvm/cmake/modules/FindGRPC.cmake
Index: llvm/cmake/modules/FindGRPC.cmake =================================================================== --- llvm/cmake/modules/FindGRPC.cmake +++ llvm/cmake/modules/FindGRPC.cmake @@ -23,11 +23,11 @@ find_program(PROTOC protoc) endif() -# Proto headers are generated in ${CMAKE_CURRENT_BINARY_DIR}. +# Proto headers are generated in ${GeneratedFilesLocation}. # Libraries that use these headers should adjust the include path. # FIXME(kirillbobyrev): Allow optional generation of gRPC code and give callers # control over it via additional parameters. -function(generate_grpc_protos LibraryName ProtoFile) +function(generate_grpc_protos LibraryName ProtoFile GeneratedFilesLocation) get_filename_component(ProtoSourceAbsolutePath "${CMAKE_CURRENT_SOURCE_DIR}/${ProtoFile}" ABSOLUTE) get_filename_component(ProtoSourcePath ${ProtoSourceAbsolutePath} PATH) @@ -35,6 +35,7 @@ set(GeneratedProtoHeader "${CMAKE_CURRENT_BINARY_DIR}/Index.pb.h") set(GeneratedGRPCSource "${CMAKE_CURRENT_BINARY_DIR}/Index.grpc.pb.cc") set(GeneratedGRPCHeader "${CMAKE_CURRENT_BINARY_DIR}/Index.grpc.pb.h") + set(${GeneratedFilesLocation} ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE) add_custom_command( OUTPUT "${GeneratedProtoSource}" "${GeneratedProtoHeader}" "${GeneratedGRPCSource}" "${GeneratedGRPCHeader}" COMMAND ${PROTOC} Index: clang-tools-extra/clangd/index/remote/server/Server.cpp =================================================================== --- clang-tools-extra/clangd/index/remote/server/Server.cpp +++ clang-tools-extra/clangd/index/remote/server/Server.cpp @@ -23,6 +23,7 @@ namespace clang { namespace clangd { +namespace remote { namespace { static const std::string Overview = R"( @@ -40,6 +41,22 @@ return loadIndex(Index, /*UseIndex=*/true); } +clangd::FuzzyFindRequest deserialize(const FuzzyFindRequest *Request) { + clangd::FuzzyFindRequest Result; + Result.Query = Request->query(); + for (const auto &Scope : Request->scopes()) + Result.Scopes.push_back(Scope); + Result.AnyScope = Request->any_scope(); + if (Request->limit()) + Result.Limit = Request->limit(); + Result.RestrictForCodeCompletion = Request->resricted_for_code_completion(); + for (const auto &Path : Request->proximity_paths()) + Result.ProximityPaths.push_back(Path); + for (const auto &Type : Request->preferred_types()) + Result.ProximityPaths.push_back(Type); + return Result; +} + class RemoteIndexServer final : public remote::Index::Service { public: RemoteIndexServer(std::unique_ptr<SymbolIndex> Index) @@ -47,24 +64,35 @@ private: grpc::Status Lookup(grpc::ServerContext *Context, - const remote::LookupRequest *Request, - grpc::ServerWriter<remote::LookupReply> *Reply) override { - llvm::outs() << "Lookup of symbol with ID " << Request->id() << '\n'; - LookupRequest Req; - auto SID = SymbolID::fromStr(Request->id()); - if (!SID) { - llvm::outs() << llvm::toString(SID.takeError()) << "\n"; - return grpc::Status::CANCELLED; + const LookupRequest *Request, + grpc::ServerWriter<Symbol> *Reply) override { + clangd::LookupRequest Req; + for (const auto &ID : Request->id()) { + const std::string ConstantString = ID; + auto SID = SymbolID::fromStr(StringRef(ConstantString)); + if (!SID) + return grpc::Status::CANCELLED; + Req.IDs.insert(*SID); } - Req.IDs.insert(*SID); - Index->lookup(Req, [&](const Symbol &Sym) { - remote::LookupReply NextSymbol; - NextSymbol.set_symbol_yaml(toYAML(Sym)); + Index->lookup(Req, [&](const clangd::Symbol &Sym) { + remote::Symbol NextSymbol; + NextSymbol.set_yaml_serializatiton(toYAML(Sym)); Reply->Write(NextSymbol); }); return grpc::Status::OK; } + grpc::Status FuzzyFind(grpc::ServerContext *Context, + const FuzzyFindRequest *Request, + FuzzyFindReply *Reply) override { + const auto Req = deserialize(Request); + bool HasMore = Index->fuzzyFind(Req, [&](const clangd::Symbol &Sym) { + Reply->add_symbols(toYAML(Sym)); + }); + Reply->set_has_more(HasMore); + return grpc::Status::OK; + } + std::unique_ptr<SymbolIndex> Index; }; @@ -83,15 +111,16 @@ } } // namespace +} // namespace remote } // namespace clangd } // namespace clang int main(int argc, char *argv[]) { - using namespace clang::clangd; - llvm::cl::ParseCommandLineOptions(argc, argv, clang::clangd::Overview); + using namespace clang::clangd::remote; + llvm::cl::ParseCommandLineOptions(argc, argv, Overview); llvm::sys::PrintStackTraceOnErrorSignal(argv[0]); - std::unique_ptr<SymbolIndex> Index = openIndex(IndexPath); + std::unique_ptr<clang::clangd::SymbolIndex> Index = openIndex(IndexPath); if (!Index) { llvm::outs() << "Failed to open the index.\n"; Index: clang-tools-extra/clangd/index/remote/client/Client.cpp =================================================================== --- clang-tools-extra/clangd/index/remote/client/Client.cpp +++ /dev/null @@ -1,91 +0,0 @@ -//===--- Client.cpp - Remote Index Client -----------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file implements a simple interactive tool which can be used to manually -// evaluate symbol search quality of Clangd index. -// -//===----------------------------------------------------------------------===// - -#include "SourceCode.h" -#include "index/Serialization.h" -#include "index/dex/Dex.h" -#include "llvm/ADT/ScopeExit.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringRef.h" -#include "llvm/ADT/StringSwitch.h" -#include "llvm/LineEditor/LineEditor.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/Signals.h" - -#include "grpcpp/grpcpp.h" - -#include "Index.grpc.pb.h" - -namespace clang { -namespace clangd { -namespace { - -llvm::cl::opt<std::string> - ServerAddress("server-address", - llvm::cl::desc("Address of remote index server to use."), - llvm::cl::init("0.0.0.0:50051")); - -static const std::string Overview = R"( -This is an **experimental** interactive tool to process user-provided search -queries over given symbol collection obtained via clangd-indexer with the help -of remote index server. The client will connect to remote index server and pass -it lookup queries. -)"; - -class RemoteIndexClient { -public: - RemoteIndexClient(std::shared_ptr<grpc::Channel> Channel) - : Stub(remote::Index::NewStub(Channel)) {} - - void lookup(llvm::StringRef ID) { - llvm::outs() << "Lookup of symbol with ID " << ID << '\n'; - remote::LookupRequest Proto; - Proto.set_id(ID.str()); - - grpc::ClientContext Context; - remote::LookupReply Reply; - std::unique_ptr<grpc::ClientReader<remote::LookupReply>> Reader( - Stub->Lookup(&Context, Proto)); - while (Reader->Read(&Reply)) { - llvm::outs() << Reply.symbol_yaml(); - } - grpc::Status Status = Reader->Finish(); - if (Status.ok()) { - llvm::outs() << "lookupRequest rpc succeeded.\n"; - } else { - llvm::outs() << "lookupRequest rpc failed.\n"; - } - } - -private: - std::unique_ptr<remote::Index::Stub> Stub; -}; - -} // namespace -} // namespace clangd -} // namespace clang - -int main(int argc, const char *argv[]) { - using namespace clang::clangd; - - llvm::cl::ParseCommandLineOptions(argc, argv, Overview); - llvm::cl::ResetCommandLineParser(); // We reuse it for REPL commands. - llvm::sys::PrintStackTraceOnErrorSignal(argv[0]); - - RemoteIndexClient IndexClient( - grpc::CreateChannel(ServerAddress, grpc::InsecureChannelCredentials())); - - llvm::LineEditor LE("remote-index-client"); - while (llvm::Optional<std::string> Request = LE.readLine()) - IndexClient.lookup(std::move(*Request)); -} Index: clang-tools-extra/clangd/index/remote/client/CMakeLists.txt =================================================================== --- clang-tools-extra/clangd/index/remote/client/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -set(LLVM_LINK_COMPONENTS - LineEditor - Support - ) -add_clang_executable(clangd-index-client - Client.cpp - ) -target_compile_definitions(clangd-index-client PRIVATE -DGOOGLE_PROTOBUF_NO_RTTI=1) -clang_target_link_libraries(clangd-index-client - PRIVATE - clangDaemon - ) -target_link_libraries(clangd-index-client - PRIVATE - RemoteIndexProtos - - protobuf - grpc++ - ) Index: clang-tools-extra/clangd/index/remote/Index.proto =================================================================== --- clang-tools-extra/clangd/index/remote/Index.proto +++ clang-tools-extra/clangd/index/remote/Index.proto @@ -11,9 +11,30 @@ package clang.clangd.remote; service Index { - rpc Lookup(LookupRequest) returns (stream LookupReply) {} + rpc Lookup(LookupRequest) returns (stream Symbol) {} + + rpc FuzzyFind(FuzzyFindRequest) returns (FuzzyFindReply) {} +} + +message LookupRequest { repeated string id = 1; } + +// FIXME(kirillbobyrev): Properly serialize symbol instead of passing YAML. +message Symbol { + string yaml_serializatiton = 1; } -message LookupRequest { string id = 1; } +message FuzzyFindRequest { + string query = 1; + repeated string scopes = 2; + bool any_scope = 3; + uint32 limit = 4; + bool resricted_for_code_completion = 5; + repeated string proximity_paths = 6; + repeated string preferred_types = 7; +} -message LookupReply { string symbol_yaml = 1; } +message FuzzyFindReply { + // FIXME(kirillbobyrev): Convert to Symbol. + repeated string symbols = 1; + bool has_more = 2; +} Index: clang-tools-extra/clangd/index/remote/Index.h =================================================================== --- /dev/null +++ clang-tools-extra/clangd/index/remote/Index.h @@ -0,0 +1,50 @@ +//===--- Index.h -------------------------------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_REMOTE_INDEX_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_REMOTE_INDEX_H + +#include "grpcpp/grpcpp.h" + +#include "Index.grpc.pb.h" +#include "index/Index.h" + +namespace clang { +namespace clangd { +namespace remote { + +class IndexClient : public SymbolIndex { +public: + IndexClient(llvm::StringRef Address) + : Stub(remote::Index::NewStub(grpc::CreateChannel( + Address.str(), grpc::InsecureChannelCredentials()))) {} + + void lookup(const clangd::LookupRequest &, + llvm::function_ref<void(const clangd::Symbol &)>) const override; + + bool + fuzzyFind(const clangd::FuzzyFindRequest &, + llvm::function_ref<void(const clangd::Symbol &)>) const override; + bool refs(const RefsRequest &, + llvm::function_ref<void(const Ref &)>) const override; + void + relations(const RelationsRequest &, + llvm::function_ref<void(const clangd::SymbolID &, + const clangd::Symbol &)>) const override; + + size_t estimateMemoryUsage() const override; + +private: + std::unique_ptr<remote::Index::Stub> Stub; +}; + +} // namespace remote +} // namespace clangd +} // namespace clang + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_REMOTE_INDEX_H Index: clang-tools-extra/clangd/index/remote/Index.cpp =================================================================== --- /dev/null +++ clang-tools-extra/clangd/index/remote/Index.cpp @@ -0,0 +1,81 @@ +//===--- Index.cpp -----------------------------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "Index.h" +#include "Index.grpc.pb.h" +#include "index/Serialization.h" +#include "llvm/Support/YAMLTraits.h" + +namespace clang { +namespace clangd { +namespace remote { + +// IndexClient does not take any space since the data is stored on the server. +size_t IndexClient::estimateMemoryUsage() const { return 0; } + +void IndexClient::lookup( + const clangd::LookupRequest &Request, + llvm::function_ref<void(const clangd::Symbol &)> Callback) const { + LookupRequest RPCRequest; + for (const auto &SymbolID : Request.IDs) + RPCRequest.add_id(SymbolID.str()); + + grpc::ClientContext Context; + remote::Symbol Reply; + std::unique_ptr<grpc::ClientReader<remote::Symbol>> Reader( + Stub->Lookup(&Context, RPCRequest)); + llvm::BumpPtrAllocator Arena; + llvm::UniqueStringSaver Strings(Arena); + while (Reader->Read(&Reply)) + Callback(fromYAML(Reply.yaml_serializatiton(), &Strings)); + grpc::Status Status = Reader->Finish(); + llvm::outs() << "lookup rpc " << (Status.ok() ? "succeeded" : "failed") + << '\n'; +} + +bool IndexClient::fuzzyFind( + const clangd::FuzzyFindRequest &Request, + llvm::function_ref<void(const clangd::Symbol &)> Callback) const { + FuzzyFindRequest RPCRequest; + RPCRequest.set_query(Request.Query); + for (const auto &Scope : Request.Scopes) + RPCRequest.add_scopes(Scope); + RPCRequest.set_any_scope(Request.AnyScope); + if (Request.Limit) + RPCRequest.set_limit(*Request.Limit); + RPCRequest.set_resricted_for_code_completion( + Request.RestrictForCodeCompletion); + for (const auto &Path : Request.ProximityPaths) + RPCRequest.add_proximity_paths(Path); + for (const auto &Type : Request.PreferredTypes) + RPCRequest.add_preferred_types(Type); + + grpc::ClientContext Context; + FuzzyFindReply Reply; + grpc::Status Status = Stub->FuzzyFind(&Context, RPCRequest, &Reply); + llvm::BumpPtrAllocator Arena; + llvm::UniqueStringSaver Strings(Arena); + for (const auto &SymbolYAML : Reply.symbols()) + Callback(fromYAML(SymbolYAML, &Strings)); + llvm::outs() << "FuzzyFind rpc " << (Status.ok() ? "succeeded" : "failed") + << '\n'; + return Reply.has_more(); +} + +bool IndexClient::refs(const RefsRequest &, + llvm::function_ref<void(const Ref &)>) const { + return false; +} +void IndexClient::relations( + const RelationsRequest &, + llvm::function_ref<void(const clangd::SymbolID &, const clangd::Symbol &)>) + const {} + +} // namespace remote +} // namespace clangd +} // namespace clang Index: clang-tools-extra/clangd/index/remote/CMakeLists.txt =================================================================== --- clang-tools-extra/clangd/index/remote/CMakeLists.txt +++ clang-tools-extra/clangd/index/remote/CMakeLists.txt @@ -1,7 +1,20 @@ -generate_grpc_protos(RemoteIndexProtos "Index.proto") +generate_grpc_protos(RemoteIndexProtos "Index.proto" RemoteProtosLocation) -include_directories("${CMAKE_CURRENT_BINARY_DIR}") +include_directories(${RemoteProtosLocation}) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../) -add_subdirectory(client) +# FIXME(kirillbobyrev): This should be target-local, but it's not working with +# add_clang_library. +add_definitions(-DGOOGLE_PROTOBUF_NO_RTTI=1) +add_clang_library(clangDaemonRemoteIndex + Index.cpp + + LINK_LIBS + RemoteIndexProtos + + protobuf + grpc++ + clangDaemon + ) + add_subdirectory(server) Index: clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp =================================================================== --- clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp +++ clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp @@ -11,9 +11,11 @@ // //===----------------------------------------------------------------------===// +#include "Index.grpc.pb.h" #include "SourceCode.h" #include "index/Serialization.h" #include "index/dex/Dex.h" +#include "index/remote/Index.h" #include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" @@ -26,12 +28,18 @@ namespace clangd { namespace { -llvm::cl::opt<std::string> IndexPath(llvm::cl::desc("<INDEX FILE>"), - llvm::cl::Positional, llvm::cl::Required); +llvm::cl::opt<std::string> IndexLocation(llvm::cl::desc("<INDEX LOCATION>"), + llvm::cl::Positional); llvm::cl::opt<std::string> ExecCommand("c", llvm::cl::desc("Command to execute and then exit")); +// FIXME(kirillbobyrev): This is not working properly for some reason. +llvm::cl::opt<bool> + RemoteMode("remote", + llvm::cl::desc("Connect to <INDEX LOCATION> remote index"), + llvm::cl::init(true)); + static const std::string Overview = R"( This is an **experimental** interactive tool to process user-provided search queries over given symbol collection obtained via clangd-indexer. The @@ -150,7 +158,7 @@ } Request.AnyScope = Request.Scopes.empty(); // FIXME(kbobyrev): Print symbol final scores to see the distribution. - static const auto OutputFormat = "{0,-4} | {1,-40} | {2,-25}\n"; + static const auto *OutputFormat = "{0,-4} | {1,-40} | {2,-25}\n"; llvm::outs() << llvm::formatv(OutputFormat, "Rank", "Symbol ID", "Symbol Name"); size_t Rank = 0; @@ -266,12 +274,13 @@ {"find", "Search for symbols with fuzzyFind", std::make_unique<FuzzyFind>}, {"lookup", "Dump symbol details by ID or qualified name", std::make_unique<Lookup>}, - {"refs", "Find references by ID or qualified name", - std::make_unique<Refs>}, + {"refs", "Find references by ID or qualified name", std::make_unique<Refs>}, }; std::unique_ptr<SymbolIndex> openIndex(llvm::StringRef Index) { - return loadIndex(Index, /*UseDex=*/true); + return RemoteMode + ? std::unique_ptr<SymbolIndex>(new remote::IndexClient(Index)) + : loadIndex(Index, /*UseDex=*/true); } bool runCommand(std::string Request, const SymbolIndex &Index) { @@ -314,9 +323,7 @@ llvm::sys::PrintStackTraceOnErrorSignal(argv[0]); std::unique_ptr<SymbolIndex> Index; - reportTime("Dex build", [&]() { - Index = openIndex(IndexPath); - }); + reportTime("Dex build", [&]() { Index = openIndex(IndexLocation); }); if (!Index) { llvm::outs() << "Failed to open the index.\n"; Index: clang-tools-extra/clangd/index/dex/dexp/CMakeLists.txt =================================================================== --- clang-tools-extra/clangd/index/dex/dexp/CMakeLists.txt +++ clang-tools-extra/clangd/index/dex/dexp/CMakeLists.txt @@ -1,5 +1,9 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../../) +include_directories(${RemoteProtosLocation}) +# FIXME(kirillbobyrev): This should be target-local, but it's not working with +# add_clang_library. +add_definitions(-DGOOGLE_PROTOBUF_NO_RTTI=1) set(LLVM_LINK_COMPONENTS LineEditor Support @@ -16,4 +20,8 @@ target_link_libraries(dexp PRIVATE clangDaemon + clangDaemonRemoteIndex + + RemoteIndexProtos + grpc++ ) Index: clang-tools-extra/clangd/index/YAMLSerialization.cpp =================================================================== --- clang-tools-extra/clangd/index/YAMLSerialization.cpp +++ clang-tools-extra/clangd/index/YAMLSerialization.cpp @@ -418,5 +418,12 @@ return Buf; } +clangd::Symbol fromYAML(StringRef YAML, llvm::UniqueStringSaver *Strings) { + clangd::Symbol Deserialized; + llvm::yaml::Input YAMLInput(YAML, Strings); + YAMLInput >> Deserialized; + return Deserialized; +} + } // namespace clangd } // namespace clang Index: clang-tools-extra/clangd/index/Serialization.h =================================================================== --- clang-tools-extra/clangd/index/Serialization.h +++ clang-tools-extra/clangd/index/Serialization.h @@ -78,6 +78,9 @@ std::string toYAML(const std::pair<SymbolID, ArrayRef<Ref>> &); std::string toYAML(const Relation &); +// Deserialize a single symbol from YAML. +clangd::Symbol fromYAML(StringRef YAML, llvm::UniqueStringSaver *Strings); + // Build an in-memory static index from an index file. // The size should be relatively small, so data can be managed in memory. std::unique_ptr<SymbolIndex> loadIndex(llvm::StringRef Filename, Index: clang-tools-extra/clangd/CMakeLists.txt =================================================================== --- clang-tools-extra/clangd/CMakeLists.txt +++ clang-tools-extra/clangd/CMakeLists.txt @@ -140,7 +140,6 @@ endif() add_subdirectory(tool) add_subdirectory(indexer) -add_subdirectory(index/dex/dexp) if (LLVM_INCLUDE_BENCHMARKS) add_subdirectory(benchmarks) @@ -162,3 +161,4 @@ include(FindGRPC) add_subdirectory(index/remote) endif() +add_subdirectory(index/dex/dexp)
_______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits