sammccall created this revision. sammccall added a reviewer: ioeric. Herald added subscribers: cfe-commits, jfb, kadircet, arphaman, jkorous, MaskRay, ilya-biryukov, mgorny.
See tinyurl.com/clangd-automatic-index for design and goals. Lots of limitations to keep this patch smallish, TODOs everywhere: - no serialization to disk - no changes to dynamic index, which now has a much simpler job - no partitioning of symbols by file to avoid duplication of header symbols - no reindexing of edited files - only a single worker thread - compilation database is slurped synchronously (doesn't scale) - uses memindex, rebuilds after every file (should be dex, periodically) Still needs tests, but should be ready for review of the basic shape. Repository: rCTE Clang Tools Extra https://reviews.llvm.org/D53032 Files: clangd/CMakeLists.txt clangd/ClangdLSPServer.cpp clangd/ClangdLSPServer.h clangd/ClangdServer.cpp clangd/Compiler.cpp clangd/Compiler.h clangd/GlobalCompilationDatabase.cpp clangd/GlobalCompilationDatabase.h clangd/index/Background.cpp clangd/index/Background.h clangd/tool/ClangdMain.cpp
Index: clangd/tool/ClangdMain.cpp =================================================================== --- clangd/tool/ClangdMain.cpp +++ clangd/tool/ClangdMain.cpp @@ -167,6 +167,14 @@ "eventually. Don't rely on it."), llvm::cl::init(""), llvm::cl::Hidden); +static llvm::cl::opt<bool> AutoIndex( + "auto-index", + llvm::cl::desc( + "Build a full index for the codebase containing edited files. " + "Indexing will occur in the background. " + "This option is still experimental, as the indexing is inefficient."), + llvm::cl::init(false), llvm::cl::Hidden); + enum CompileArgsFrom { LSPCompileArgs, FilesystemCompileArgs }; static llvm::cl::opt<CompileArgsFrom> CompileArgsFrom( "compile_args_from", llvm::cl::desc("The source of compile commands"), @@ -316,9 +324,10 @@ CCOpts.AllScopes = AllScopesCompletion; // Initialize and run ClangdLSPServer. - ClangdLSPServer LSPServer( - Out, CCOpts, CompileCommandsDirPath, - /*ShouldUseInMemoryCDB=*/CompileArgsFrom == LSPCompileArgs, Opts); + ClangdLSPServer LSPServer(Out, CCOpts, CompileCommandsDirPath, + /*ShouldUseInMemoryCDB=*/CompileArgsFrom == + LSPCompileArgs, + AutoIndex, Opts); constexpr int NoShutdownRequestErrorCode = 1; llvm::set_thread_name("clangd.main"); // Change stdin to binary to not lose \r\n on windows. Index: clangd/index/Background.h =================================================================== --- /dev/null +++ clangd/index/Background.h @@ -0,0 +1,73 @@ +//===--- Background.h - Build an index in a background thread ----*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_BACKGROUND_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_BACKGROUND_H + +#include "Context.h" +#include "FSProvider.h" +#include "index/Index.h" +#include "index/FileIndex.h" +#include "clang/Tooling/CompilationDatabase.h" +#include "llvm/Support/SHA1.h" +#include <condition_variable> +#include <deque> +#include <thread> + +namespace clang { +namespace clangd { + +// Builds an in-memory index by by running the static indexer action over +// all commands in a compilation database. Indexing happens in the background. +// TODO: it should also persist its state on disk for fast start. +class BackgroundIndex : public SwapIndex { +public: + // TODO: FileSystemProvider is not const-correct. + // TODO: resource-dir injection should be hoisted somewhere common. + BackgroundIndex(Context BackgroundContext, + StringRef ResourceDir, FileSystemProvider *); + ~BackgroundIndex(); // Blocks while the current task finishes. + + // Index all TUs described in the compilation database. + // The indexing happens in a background thread, so after enqueueing files + // for indexing their symbols will be available sometime later. + void enqueueAll(llvm::StringRef Directory, + const tooling::CompilationDatabase &); + + // Cause background threads to stop after ther current task, any remaining + // tasks will be discarded. + void stop(); + +private: + // configuration + std::string ResourceDir; + FileSystemProvider *FSProvider; + Context BackgroundContext; + + // index state + llvm::Error index(tooling::CompileCommand); + FileSymbols IndexedSymbols; // Index contents. + using Hash = decltype(llvm::SHA1::hash({})); + llvm::StringMap<Hash> FileHash; // Digest of indexed file. + + // queue management + using Task = std::function<void()>; // TODO: use multiple worker threads. + void run(); // Main loop executed by Thread. Runs tasks from Queue. + void enqueueLocked(tooling::CompileCommand Cmd); + std::thread Thread; + std::mutex QueueMu; + std::condition_variable QueueCV; + bool ShouldStop = false; + std::deque<Task> Queue; +}; + +} // namespace clangd +} // namespace clang + +#endif Index: clangd/index/Background.cpp =================================================================== --- /dev/null +++ clangd/index/Background.cpp @@ -0,0 +1,168 @@ +//===-- Background.cpp - Build an index in a background thread ------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "index/Background.h" +#include "index/MemIndex.h" +#include "index/Serialization.h" +#include "index/IndexAction.h" +#include "Compiler.h" +#include "ClangdUnit.h" +#include "Logger.h" +#include "Trace.h" +#include "llvm/Support/SHA1.h" +#include <random> + +using namespace llvm; +namespace clang { +namespace clangd { + +BackgroundIndex::BackgroundIndex(Context BackgroundContext, + StringRef ResourceDir, + FileSystemProvider *FSProvider) + : SwapIndex(llvm::make_unique<MemIndex>()), ResourceDir(ResourceDir), + FSProvider(FSProvider), BackgroundContext(std::move(BackgroundContext)), + Thread([this] { run(); }) {} + +BackgroundIndex::~BackgroundIndex() { + stop(); + Thread.join(); +} + +void BackgroundIndex::stop() { + { + std::lock_guard<std::mutex> Lock(QueueMu); + ShouldStop = true; + } + QueueCV.notify_all(); +} + +void BackgroundIndex::run() { + WithContext Background(std::move(BackgroundContext)); + while(true) { + llvm::Optional<Task> Task; + { + std::unique_lock<std::mutex> Lock(QueueMu); + QueueCV.wait(Lock, [&] { return ShouldStop || !Queue.empty(); }); + if (ShouldStop) + return; + Task = std::move(Queue.front()); + Queue.pop_front(); + } + (*Task)(); + } +} + +void BackgroundIndex::enqueueAll(StringRef Directory, + const tooling::CompilationDatabase &CDB) { + // TODO: this function may be slow. Perhaps enqueue a task to re-read the CDB + // from disk and enqueue the commands asynchronously? + auto Cmds = CDB.getAllCompileCommands(); + std::mt19937 Generator(std::random_device{}()); + std::shuffle(Cmds.begin(), Cmds.end(), Generator); + log("Enqueueing {0} commands for indexing from {1}", Cmds.size(), Directory); + { + std::lock_guard<std::mutex> Lock(QueueMu); + for (auto &Cmd : Cmds) + enqueueLocked(std::move(Cmd)); + } + QueueCV.notify_all(); +} + +void BackgroundIndex::enqueueLocked(tooling::CompileCommand Cmd) { + Queue.push_back(Bind( + [this](tooling::CompileCommand Cmd) { + std::string Filename = Cmd.Filename; + Cmd.CommandLine.push_back("-resource-dir=" + ResourceDir); + if (auto Error = index(std::move(Cmd))) + log("Indexing {0} failed: {1}", Filename, std::move(Error)); + }, + std::move(Cmd))); +} + +llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd) { + trace::Span Tracer("BackgroundIndex"); + SPAN_ATTACH(Tracer, "file", Cmd.Filename); + SmallString<128> AbsolutePath; + if (llvm::sys::path::is_absolute(Cmd.Filename)) { + AbsolutePath = Cmd.Filename; + } else { + AbsolutePath = Cmd.Directory; + llvm::sys::path::append(AbsolutePath, Cmd.Filename); + } + llvm::sys::path::native(AbsolutePath); + + auto FS = FSProvider->getFileSystem(); + auto Buf = FS->getBufferForFile(AbsolutePath); + if (!Buf) + return errorCodeToError(Buf.getError()); + StringRef Contents = Buf->get()->getBuffer(); + auto Hash = SHA1::hash({(const uint8_t *)Contents.data(), Contents.size()}); + + if (FileHash.lookup(AbsolutePath) == Hash) { + vlog("No need to index {0}, already up to date", AbsolutePath); + return Error::success(); + } + + log("Indexing {0}", Cmd.Filename, toHex(Hash)); + ParseInputs Inputs; + Inputs.FS = std::move(FS); + Inputs.FS->setCurrentWorkingDirectory(Cmd.Directory); + Inputs.CompileCommand = std::move(Cmd); + auto CI = buildCompilerInvocation(Inputs); + if (!CI) + return createStringError(llvm::inconvertibleErrorCode(), + "Couldn't build compiler invocation"); + IgnoreDiagnostics IgnoreDiags; + auto Clang = prepareCompilerInstance( + std::move(CI), /*Preamble=*/nullptr, std::move(*Buf), + std::make_shared<PCHContainerOperations>(), Inputs.FS, IgnoreDiags); + if (!Clang) + return createStringError(llvm::inconvertibleErrorCode(), + "Couldn't build compiler instance"); + + SymbolCollector::Options IndexOpts; + SymbolSlab Symbols; + RefSlab Refs; + IndexFileIn IndexData; + auto Action = createStaticIndexingAction( + IndexOpts, [&](SymbolSlab S) { Symbols = std::move(S); }, + [&](RefSlab R) { Refs = std::move(R); }); + + // We're going to run clang here, and it could potentially crash. + // We could use CrashRecoveryContext to try to make indexing crashes nonfatal, + // but the leaky "recovery" is pretty scary too in a long-running process. + // If crashes are a real problem, maybe we should fork a child process. + + const FrontendInputFile &Input = Clang->getFrontendOpts().Inputs.front(); + if (!Action->BeginSourceFile(*Clang, Input)) + return createStringError(llvm::inconvertibleErrorCode(), + "BeginSourceFile() failed"); + if (!Action->Execute()) + return createStringError(llvm::inconvertibleErrorCode(), + "Execute() failed"); + Action->EndSourceFile(); + + log("Indexed {0} ({1} symbols, {2} refs)", Inputs.CompileCommand.Filename, + Symbols.size(), Refs.size()); + SPAN_ATTACH(Tracer, "symbols", int(Symbols.size())); + SPAN_ATTACH(Tracer, "refs", int(Refs.size())); + // TODO: partition the symbols by file rather than TU, to avoid duplication. + IndexedSymbols.update(AbsolutePath, + llvm::make_unique<SymbolSlab>(std::move(Symbols)), + llvm::make_unique<RefSlab>(std::move(Refs))); + FileHash[AbsolutePath] = Hash; + + // TODO: this should rebuild once-in-a-while, not after every file. + vlog("Rebuilding automatic index"); + reset(IndexedSymbols.buildMemIndex()); + return Error::success(); +} + +} // namespace clangd +} // namespace clang Index: clangd/GlobalCompilationDatabase.h =================================================================== --- clangd/GlobalCompilationDatabase.h +++ clangd/GlobalCompilationDatabase.h @@ -51,7 +51,10 @@ : public GlobalCompilationDatabase { public: DirectoryBasedGlobalCompilationDatabase( - llvm::Optional<Path> CompileCommandsDir); + llvm::Optional<Path> CompileCommandsDir, + std::function<void(llvm::StringRef, + const clang::tooling::CompilationDatabase &)> + OnNewCDB = nullptr); ~DirectoryBasedGlobalCompilationDatabase() override; /// Scans File's parents looking for compilation databases. @@ -84,6 +87,8 @@ /// Used for command argument pointing to folder where compile_commands.json /// is located. llvm::Optional<Path> CompileCommandsDir; + std::function<void(llvm::StringRef, const tooling::CompilationDatabase &)> + OnNewCDB; }; /// A wrapper around GlobalCompilationDatabase that caches the compile commands. Index: clangd/GlobalCompilationDatabase.cpp =================================================================== --- clangd/GlobalCompilationDatabase.cpp +++ clangd/GlobalCompilationDatabase.cpp @@ -32,8 +32,12 @@ DirectoryBasedGlobalCompilationDatabase:: DirectoryBasedGlobalCompilationDatabase( - llvm::Optional<Path> CompileCommandsDir) - : CompileCommandsDir(std::move(CompileCommandsDir)) {} + llvm::Optional<Path> CompileCommandsDir, + std::function<void(llvm::StringRef, + const tooling::CompilationDatabase &)> + OnNewCDB) + : CompileCommandsDir(std::move(CompileCommandsDir)), + OnNewCDB(std::move(OnNewCDB)) {} DirectoryBasedGlobalCompilationDatabase:: ~DirectoryBasedGlobalCompilationDatabase() = default; @@ -97,6 +101,11 @@ auto CDB = tooling::CompilationDatabase::loadFromDirectory(Dir, Error); auto Result = CDB.get(); CompilationDatabases.insert(std::make_pair(Dir, std::move(CDB))); + if (Result && OnNewCDB) { + Mutex.unlock(); + OnNewCDB(Dir, *Result); + Mutex.lock(); + } return Result; } Index: clangd/Compiler.h =================================================================== --- clangd/Compiler.h +++ clangd/Compiler.h @@ -48,6 +48,8 @@ std::shared_ptr<PCHContainerOperations>, IntrusiveRefCntPtr<vfs::FileSystem>, DiagnosticConsumer &); +std::string getStandardResourceDir(); + } // namespace clangd } // namespace clang Index: clangd/Compiler.cpp =================================================================== --- clangd/Compiler.cpp +++ clangd/Compiler.cpp @@ -80,5 +80,10 @@ return Clang; } +std::string getStandardResourceDir() { + static int Dummy; // Just an address in this process. + return CompilerInvocation::GetResourcesPath("clangd", (void *)&Dummy); +} + } // namespace clangd } // namespace clang Index: clangd/ClangdServer.cpp =================================================================== --- clangd/ClangdServer.cpp +++ clangd/ClangdServer.cpp @@ -9,6 +9,7 @@ #include "ClangdServer.h" #include "CodeComplete.h" +#include "Compiler.h" #include "FindSymbols.h" #include "Headers.h" #include "SourceCode.h" @@ -43,11 +44,6 @@ handleAllErrors(std::move(Err), [](const llvm::ErrorInfoBase &) {}); } -std::string getStandardResourceDir() { - static int Dummy; // Just an address in this process. - return CompilerInvocation::GetResourcesPath("clangd", (void *)&Dummy); -} - class RefactoringResultCollector final : public tooling::RefactoringResultConsumer { public: Index: clangd/ClangdLSPServer.h =================================================================== --- clangd/ClangdLSPServer.h +++ clangd/ClangdLSPServer.h @@ -17,6 +17,7 @@ #include "Path.h" #include "Protocol.h" #include "ProtocolHandlers.h" +#include "index/Background.h" #include "clang/Tooling/Core/Replacement.h" #include "llvm/ADT/Optional.h" #include <memory> @@ -39,7 +40,8 @@ /// for compile_commands.json in all parent directories of each file. ClangdLSPServer(JSONOutput &Out, const clangd::CodeCompleteOptions &CCOpts, llvm::Optional<Path> CompileCommandsDir, - bool ShouldUseInMemoryCDB, const ClangdServer::Options &Opts); + bool ShouldUseInMemoryCDB, bool AutoIndex, + const ClangdServer::Options &Opts); /// Run LSP server loop, receiving input for it from \p In. \p In must be /// opened in binary mode. Output will be written using Out variable passed to @@ -110,8 +112,10 @@ class CompilationDB { public: static CompilationDB makeInMemory(); - static CompilationDB - makeDirectoryBased(llvm::Optional<Path> CompileCommandsDir); + static CompilationDB makeDirectoryBased( + llvm::Optional<Path> CompileCommandsDir, + std::function<void(llvm::StringRef, + const tooling::CompilationDatabase &)>); void invalidate(PathRef File); @@ -167,6 +171,8 @@ /// The supported completion item kinds of the client. CompletionItemKindBitset SupportedCompletionItemKinds; + llvm::Optional<BackgroundIndex> BackgroundIdx; + // Store of the current versions of the open documents. DraftStore DraftMgr; Index: clangd/ClangdLSPServer.cpp =================================================================== --- clangd/ClangdLSPServer.cpp +++ clangd/ClangdLSPServer.cpp @@ -8,6 +8,7 @@ //===----------------------------------------------------------------------===// #include "ClangdLSPServer.h" +#include "Compiler.h" #include "Diagnostics.h" #include "JSONRPCDispatcher.h" #include "SourceCode.h" @@ -149,6 +150,8 @@ void ClangdLSPServer::onShutdown(ShutdownParams &Params) { // Do essentially nothing, just say we're ready to exit. ShutdownRequestReceived = true; + if (BackgroundIdx) + BackgroundIdx->stop(); reply(nullptr); } @@ -472,15 +475,32 @@ ClangdLSPServer::ClangdLSPServer(JSONOutput &Out, const clangd::CodeCompleteOptions &CCOpts, llvm::Optional<Path> CompileCommandsDir, - bool ShouldUseInMemoryCDB, + bool ShouldUseInMemoryCDB, bool AutoIndex, const ClangdServer::Options &Opts) - : Out(Out), CDB(ShouldUseInMemoryCDB ? CompilationDB::makeInMemory() - : CompilationDB::makeDirectoryBased( - std::move(CompileCommandsDir))), + : Out(Out), CDB(ShouldUseInMemoryCDB + ? CompilationDB::makeInMemory() + : CompilationDB::makeDirectoryBased( + std::move(CompileCommandsDir), + [&](StringRef Dir, + const tooling::CompilationDatabase &CDB) { + if (BackgroundIdx) + BackgroundIdx->enqueueAll(Dir, CDB); + })), CCOpts(CCOpts), SupportedSymbolKinds(defaultSymbolKinds()), - SupportedCompletionItemKinds(defaultCompletionItemKinds()), - Server(new ClangdServer(CDB.getCDB(), FSProvider, /*DiagConsumer=*/*this, - Opts)) {} + SupportedCompletionItemKinds(defaultCompletionItemKinds()) { + ClangdServer::Options SOpts = Opts; + if (AutoIndex) { + log("Static index will be built in the background from compilation DB."); + BackgroundIdx.emplace(Context::empty(), + Opts.ResourceDir + ? *Opts.ResourceDir + : StringRef(getStandardResourceDir()), + &FSProvider); + SOpts.StaticIndex = BackgroundIdx.getPointer(); + } + Server.reset(new ClangdServer(CDB.getCDB(), FSProvider, + /*DiagConsumer=*/*this, SOpts)); +} bool ClangdLSPServer::run(std::FILE *In, JSONStreamStyle InputStyle) { assert(!IsDone && "Run was called before"); @@ -585,9 +605,11 @@ ClangdLSPServer::CompilationDB ClangdLSPServer::CompilationDB::makeDirectoryBased( - llvm::Optional<Path> CompileCommandsDir) { + llvm::Optional<Path> CompileCommandsDir, + std::function<void(StringRef, const tooling::CompilationDatabase &)> + OnNewCDB) { auto CDB = llvm::make_unique<DirectoryBasedGlobalCompilationDatabase>( - std::move(CompileCommandsDir)); + std::move(CompileCommandsDir), std::move(OnNewCDB)); auto CachingCDB = llvm::make_unique<CachingCompilationDb>(*CDB); return CompilationDB(std::move(CDB), std::move(CachingCDB), /*IsDirectoryBased=*/true); Index: clangd/CMakeLists.txt =================================================================== --- clangd/CMakeLists.txt +++ clangd/CMakeLists.txt @@ -38,6 +38,7 @@ URI.cpp XRefs.cpp + index/Background.cpp index/CanonicalIncludes.cpp index/FileIndex.cpp index/Index.cpp
_______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits