ilya-biryukov updated this revision to Diff 149073. ilya-biryukov marked 2 inline comments as done. ilya-biryukov added a comment.
- Added a unit test - Address review comments - Add ASTRetentionPolicy param to ClangdServer Repository: rCTE Clang Tools Extra https://reviews.llvm.org/D47063 Files: clangd/ClangdServer.cpp clangd/ClangdServer.h clangd/ClangdUnit.cpp clangd/ClangdUnit.h clangd/TUScheduler.cpp clangd/TUScheduler.h test/clangd/trace.test unittests/clangd/FileIndexTests.cpp unittests/clangd/TUSchedulerTests.cpp
Index: unittests/clangd/TUSchedulerTests.cpp =================================================================== --- unittests/clangd/TUSchedulerTests.cpp +++ unittests/clangd/TUSchedulerTests.cpp @@ -18,8 +18,11 @@ namespace clang { namespace clangd { +using ::testing::_; +using ::testing::AnyOf; using ::testing::Pair; using ::testing::Pointee; +using ::testing::UnorderedElementsAre; void ignoreUpdate(llvm::Optional<std::vector<Diag>>) {} void ignoreError(llvm::Error Err) { @@ -43,7 +46,8 @@ TUScheduler S(getDefaultAsyncThreadsCount(), /*StorePreamblesInMemory=*/true, /*PreambleParsedCallback=*/nullptr, - /*UpdateDebounce=*/std::chrono::steady_clock::duration::zero()); + /*UpdateDebounce=*/std::chrono::steady_clock::duration::zero(), + ASTRetentionPolicy()); auto Added = testPath("added.cpp"); Files[Added] = ""; @@ -99,7 +103,8 @@ getDefaultAsyncThreadsCount(), /*StorePreamblesInMemory=*/true, /*PreambleParsedCallback=*/nullptr, - /*UpdateDebounce=*/std::chrono::steady_clock::duration::zero()); + /*UpdateDebounce=*/std::chrono::steady_clock::duration::zero(), + ASTRetentionPolicy()); auto Path = testPath("foo.cpp"); S.update(Path, getInputs(Path, ""), WantDiagnostics::Yes, [&](std::vector<Diag>) { Ready.wait(); }); @@ -127,7 +132,8 @@ TUScheduler S(getDefaultAsyncThreadsCount(), /*StorePreamblesInMemory=*/true, /*PreambleParsedCallback=*/nullptr, - /*UpdateDebounce=*/std::chrono::seconds(1)); + /*UpdateDebounce=*/std::chrono::seconds(1), + ASTRetentionPolicy()); // FIXME: we could probably use timeouts lower than 1 second here. auto Path = testPath("foo.cpp"); S.update(Path, getInputs(Path, "auto (debounced)"), WantDiagnostics::Auto, @@ -158,7 +164,8 @@ TUScheduler S(getDefaultAsyncThreadsCount(), /*StorePreamblesInMemory=*/true, /*PreambleParsedCallback=*/nullptr, - /*UpdateDebounce=*/std::chrono::milliseconds(50)); + /*UpdateDebounce=*/std::chrono::milliseconds(50), + ASTRetentionPolicy()); std::vector<std::string> Files; for (int I = 0; I < FilesCount; ++I) { @@ -219,18 +226,18 @@ { WithContextValue WithNonce(NonceKey, ++Nonce); - S.runWithPreamble( - "CheckPreamble", File, - [Inputs, Nonce, &Mut, &TotalPreambleReads]( - llvm::Expected<InputsAndPreamble> Preamble) { - EXPECT_THAT(Context::current().get(NonceKey), Pointee(Nonce)); - - ASSERT_TRUE((bool)Preamble); - EXPECT_EQ(Preamble->Contents, Inputs.Contents); - - std::lock_guard<std::mutex> Lock(Mut); - ++TotalPreambleReads; - }); + S.runWithPreamble("CheckPreamble", File, + [Inputs, Nonce, &Mut, &TotalPreambleReads]( + llvm::Expected<InputsAndPreamble> Preamble) { + EXPECT_THAT(Context::current().get(NonceKey), + Pointee(Nonce)); + + ASSERT_TRUE((bool)Preamble); + EXPECT_EQ(Preamble->Contents, Inputs.Contents); + + std::lock_guard<std::mutex> Lock(Mut); + ++TotalPreambleReads; + }); } } } @@ -242,5 +249,55 @@ EXPECT_EQ(TotalPreambleReads, FilesCount * UpdatesPerFile); } +TEST_F(TUSchedulerTests, EvictedAST) { + ASTRetentionPolicy Policy; + Policy.MaxRetainedASTs = 2; + TUScheduler S( + /*AsyncThreadsCount=*/1, /*StorePreambleInMemory=*/true, + PreambleParsedCallback(), + /*UpdateDebounce=*/std::chrono::steady_clock::duration::zero(), Policy); + + llvm::StringLiteral SourceContents = R"cpp( + int* a; + double* b = a; + )cpp"; + + auto Foo = testPath("foo.cpp"); + auto Bar = testPath("bar.cpp"); + auto Baz = testPath("baz.cpp"); + + std::atomic<int> BuiltASTCounter; + BuiltASTCounter = false; + // Build one file in advance. We will not access it later, so it will be the + // one that the cache will evict. + S.update(Foo, getInputs(Foo, SourceContents), WantDiagnostics::Yes, + [&BuiltASTCounter](std::vector<Diag> Diags) { ++BuiltASTCounter; }); + ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(1))); + ASSERT_EQ(BuiltASTCounter.load(), 1); + + // Build two more files. Since we can retain only 2 ASTs, these should be the + // ones we see in the cache later. + S.update(Bar, getInputs(Bar, SourceContents), WantDiagnostics::Yes, + [&BuiltASTCounter](std::vector<Diag> Diags) { ++BuiltASTCounter; }); + S.update(Baz, getInputs(Baz, SourceContents), WantDiagnostics::Yes, + [&BuiltASTCounter](std::vector<Diag> Diags) { ++BuiltASTCounter; }); + ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(1))); + ASSERT_EQ(BuiltASTCounter.load(), 3); + + // Check only the last two ASTs are retained. + ASSERT_THAT(S.getFilesWithCachedAST(), UnorderedElementsAre(Bar, Baz)); + + // Access the old file again. + S.update(Foo, getInputs(Foo, SourceContents), WantDiagnostics::Yes, + [&BuiltASTCounter](std::vector<Diag> Diags) { ++BuiltASTCounter; }); + ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(1))); + ASSERT_EQ(BuiltASTCounter.load(), 4); + + // Check the AST for foo.cpp is retained now and one of the others got + // evicted. + EXPECT_THAT(S.getFilesWithCachedAST(), + UnorderedElementsAre(Foo, AnyOf(Bar, Baz))); +} + } // namespace clangd } // namespace clang Index: unittests/clangd/FileIndexTests.cpp =================================================================== --- unittests/clangd/FileIndexTests.cpp +++ unittests/clangd/FileIndexTests.cpp @@ -11,6 +11,7 @@ #include "TestFS.h" #include "TestTU.h" #include "index/FileIndex.h" +#include "clang/Frontend/CompilerInvocation.h" #include "clang/Frontend/PCHContainerOperations.h" #include "clang/Lex/Preprocessor.h" #include "clang/Tooling/CompilationDatabase.h" @@ -208,18 +209,6 @@ TEST(FileIndexTest, RebuildWithPreamble) { auto FooCpp = testPath("foo.cpp"); auto FooH = testPath("foo.h"); - FileIndex Index; - bool IndexUpdated = false; - CppFile File("foo.cpp", /*StorePreambleInMemory=*/true, - std::make_shared<PCHContainerOperations>(), - [&Index, &IndexUpdated](PathRef FilePath, ASTContext &Ctx, - std::shared_ptr<Preprocessor> PP) { - EXPECT_FALSE(IndexUpdated) - << "Expected only a single index update"; - IndexUpdated = true; - Index.update(FilePath, &Ctx, std::move(PP)); - }); - // Preparse ParseInputs. ParseInputs PI; PI.CompileCommand.Directory = testRoot(); @@ -243,7 +232,19 @@ )cpp"; // Rebuild the file. - File.rebuild(std::move(PI)); + auto CI = buildCompilerInvocation(PI); + + FileIndex Index; + bool IndexUpdated = false; + buildPreamble( + FooCpp, *CI, /*OldPreamble=*/nullptr, tooling::CompileCommand(), PI, + std::make_shared<PCHContainerOperations>(), /*StoreInMemory=*/true, + [&Index, &IndexUpdated](PathRef FilePath, ASTContext &Ctx, + std::shared_ptr<Preprocessor> PP) { + EXPECT_FALSE(IndexUpdated) << "Expected only a single index update"; + IndexUpdated = true; + Index.update(FilePath, &Ctx, std::move(PP)); + }); ASSERT_TRUE(IndexUpdated); // Check the index contains symbols from the preamble, but not from the main Index: test/clangd/trace.test =================================================================== --- test/clangd/trace.test +++ test/clangd/trace.test @@ -8,14 +8,14 @@ # CHECK: "args": { # CHECK: "File": "{{.*(/|\\)}}foo.c" # CHECK: }, -# CHECK: "name": "Preamble", +# CHECK: "name": "BuildPreamble", # CHECK: "ph": "X", # CHECK: } # CHECK: { # CHECK: "args": { # CHECK: "File": "{{.*(/|\\)}}foo.c" # CHECK: }, -# CHECK: "name": "Build", +# CHECK: "name": "BuildAST", # CHECK: "ph": "X", # CHECK: } # CHECK: }, Index: clangd/TUScheduler.h =================================================================== --- clangd/TUScheduler.h +++ clangd/TUScheduler.h @@ -42,6 +42,15 @@ /// within a bounded amount of time. }; +/// Configuration of the AST retention policy. This only covers retention of +/// *idle* ASTs. If queue has operations requiring the AST, they might be +/// kept in memory. +struct ASTRetentionPolicy { + /// Maximum number of ASTs to be retained in memory when there are no pending + /// requests for them. + unsigned MaxRetainedASTs = 3; +}; + /// Handles running tasks for ClangdServer and managing the resources (e.g., /// preambles and ASTs) for opened files. /// TUScheduler is not thread-safe, only one thread should be providing updates @@ -53,13 +62,19 @@ public: TUScheduler(unsigned AsyncThreadsCount, bool StorePreamblesInMemory, PreambleParsedCallback PreambleCallback, - std::chrono::steady_clock::duration UpdateDebounce); + std::chrono::steady_clock::duration UpdateDebounce, + ASTRetentionPolicy RetentionPolicy); ~TUScheduler(); /// Returns estimated memory usage for each of the currently open files. /// The order of results is unspecified. std::vector<std::pair<Path, std::size_t>> getUsedBytesPerFile() const; + /// Returns a list of files with ASTs currently stored in memory. This method + /// is not very reliable and is only used for test. E.g., the results will not + /// contain files that currently run something over their AST. + std::vector<Path> getFilesWithCachedAST() const; + /// Schedule an update for \p File. Adds \p File to a list of tracked files if /// \p File was not part of it before. /// FIXME(ibiryukov): remove the callback from this function. @@ -99,11 +114,18 @@ /// This class stores per-file data in the Files map. struct FileData; +public: + /// Responsible for retaining and rebuilding idle ASTs. An implementation is + /// an LRU cache. + class ASTCache; + +private: const bool StorePreamblesInMemory; const std::shared_ptr<PCHContainerOperations> PCHOps; const PreambleParsedCallback PreambleCallback; Semaphore Barrier; llvm::StringMap<std::unique_ptr<FileData>> Files; + std::unique_ptr<ASTCache> IdleASTs; // None when running tasks synchronously and non-None when running tasks // asynchronously. llvm::Optional<AsyncTaskRunner> PreambleTasks; Index: clangd/TUScheduler.cpp =================================================================== --- clangd/TUScheduler.cpp +++ clangd/TUScheduler.cpp @@ -45,16 +45,89 @@ #include "TUScheduler.h" #include "Logger.h" #include "Trace.h" +#include "clang/Frontend/CompilerInvocation.h" #include "clang/Frontend/PCHContainerOperations.h" +#include "llvm/ADT/ScopeExit.h" #include "llvm/Support/Errc.h" #include "llvm/Support/Path.h" +#include <algorithm> #include <memory> #include <queue> #include <thread> namespace clang { namespace clangd { using std::chrono::steady_clock; + +namespace { +class ASTWorker; +} + +/// An LRU cache of idle ASTs. +/// Because we want to limit the overall number of these we retain, the cache +/// owns ASTs (and may evict them) while their workers are idle. +/// Workers borrow ASTs when active, and return them when done. +class TUScheduler::ASTCache { +public: + using Key = const ASTWorker *; + + ASTCache(unsigned MaxRetainedASTs) : MaxRetainedASTs(MaxRetainedASTs) {} + + /// Returns result of getUsedBytes() for the AST cached by \p K. + /// If no AST is cached, 0 is returned. + bool getUsedBytes(Key K) { + std::lock_guard<std::mutex> Lock(Mut); + auto It = findByKey(K); + if (It == LRU.end() || !It->second) + return 0; + return It->second->getUsedBytes(); + } + + /// Store the value in the pool, possibly removing the last used AST. + /// The value should not be in the pool when this function is called. + void put(Key K, std::unique_ptr<ParsedAST> V) { + std::unique_lock<std::mutex> Lock(Mut); + assert(findByKey(K) == LRU.end()); + + LRU.insert(LRU.begin(), {K, std::move(V)}); + if (LRU.size() <= MaxRetainedASTs) + return; + // We're past the limit, remove the last element. + std::unique_ptr<ParsedAST> ForCleanup = std::move(LRU.back().second); + LRU.pop_back(); + // Run the expensive destructor outside the lock. + Lock.unlock(); + ForCleanup.reset(); + } + + /// Returns the cached value for \p K, or llvm::None if the value is not in + /// the cache anymore. If nullptr was cached for \p K, this function will + /// return a null unique_ptr wrapped into an optional. + llvm::Optional<std::unique_ptr<ParsedAST>> take(Key K) { + std::unique_lock<std::mutex> Lock(Mut); + auto Existing = findByKey(K); + if (Existing == LRU.end()) + return llvm::None; + std::unique_ptr<ParsedAST> V = std::move(Existing->second); + LRU.erase(Existing); + return V; + } + +private: + using KVPair = std::pair<Key, std::unique_ptr<ParsedAST>>; + + std::vector<KVPair>::iterator findByKey(Key K) { + return std::find_if(LRU.begin(), LRU.end(), + [K](const KVPair &P) { return P.first == K; }); + } + + std::mutex Mut; + unsigned MaxRetainedASTs; + /// Items sorted in LRU order, i.e. first item is the most recently accessed + /// one. + std::vector<KVPair> LRU; /* GUARDED_BY(Mut) */ +}; + namespace { class ASTWorkerHandle; @@ -70,18 +143,26 @@ /// worker. class ASTWorker { friend class ASTWorkerHandle; - ASTWorker(llvm::StringRef File, Semaphore &Barrier, CppFile AST, bool RunSync, - steady_clock::duration UpdateDebounce); + ASTWorker(PathRef FileName, TUScheduler::ASTCache &LRUCache, + Semaphore &Barrier, bool RunSync, + steady_clock::duration UpdateDebounce, + std::shared_ptr<PCHContainerOperations> PCHs, + bool StorePreamblesInMemory, + PreambleParsedCallback PreambleCallback); public: /// Create a new ASTWorker and return a handle to it. /// The processing thread is spawned using \p Tasks. However, when \p Tasks /// is null, all requests will be processed on the calling thread /// synchronously instead. \p Barrier is acquired when processing each /// request, it is be used to limit the number of actively running threads. - static ASTWorkerHandle Create(llvm::StringRef File, AsyncTaskRunner *Tasks, - Semaphore &Barrier, CppFile AST, - steady_clock::duration UpdateDebounce); + static ASTWorkerHandle Create(PathRef FileName, + TUScheduler::ASTCache &IdleASTs, + AsyncTaskRunner *Tasks, Semaphore &Barrier, + steady_clock::duration UpdateDebounce, + std::shared_ptr<PCHContainerOperations> PCHs, + bool StorePreamblesInMemory, + PreambleParsedCallback PreambleCallback); ~ASTWorker(); void update(ParseInputs Inputs, WantDiagnostics, @@ -92,6 +173,7 @@ std::shared_ptr<const PreambleData> getPossiblyStalePreamble() const; std::size_t getUsedBytes() const; + bool isASTCached() const; private: // Must be called exactly once on processing thread. Will return after @@ -119,22 +201,30 @@ llvm::Optional<WantDiagnostics> UpdateType; }; - const std::string File; + /// Handles retention of ASTs. + TUScheduler::ASTCache &IdleASTs; const bool RunSync; - // Time to wait after an update to see whether another update obsoletes it. + /// Time to wait after an update to see whether another update obsoletes it. const steady_clock::duration UpdateDebounce; + /// File that ASTWorker is reponsible for. + const Path FileName; + /// Whether to keep the built preambles in memory or on disk. + const bool StorePreambleInMemory; + /// Callback, passed to the preamble builder. + const PreambleParsedCallback PreambleCallback; + /// Helper class required to build the ASTs. + const std::shared_ptr<PCHContainerOperations> PCHs; Semaphore &Barrier; - // AST and FileInputs are only accessed on the processing thread from run(). - CppFile AST; - // Inputs, corresponding to the current state of AST. + /// Inputs, corresponding to the current state of AST. ParseInputs FileInputs; - // Guards members used by both TUScheduler and the worker thread. + /// CompilerInvocation used for FileInputs. + std::unique_ptr<CompilerInvocation> Invocation; + /// Size of the last AST + /// Guards members used by both TUScheduler and the worker thread. mutable std::mutex Mutex; std::shared_ptr<const PreambleData> LastBuiltPreamble; /* GUARDED_BY(Mutex) */ - // Result of getUsedBytes() after the last rebuild or read of AST. - std::size_t LastASTSize; /* GUARDED_BY(Mutex) */ - // Set to true to signal run() to finish processing. + /// Set to true to signal run() to finish processing. bool Done; /* GUARDED_BY(Mutex) */ std::deque<Request> Requests; /* GUARDED_BY(Mutex) */ mutable std::condition_variable RequestsCV; @@ -182,27 +272,37 @@ std::shared_ptr<ASTWorker> Worker; }; -ASTWorkerHandle ASTWorker::Create(llvm::StringRef File, AsyncTaskRunner *Tasks, - Semaphore &Barrier, CppFile AST, - steady_clock::duration UpdateDebounce) { +ASTWorkerHandle ASTWorker::Create(PathRef FileName, + TUScheduler::ASTCache &IdleASTs, + AsyncTaskRunner *Tasks, Semaphore &Barrier, + steady_clock::duration UpdateDebounce, + std::shared_ptr<PCHContainerOperations> PCHs, + bool StorePreamblesInMemory, + PreambleParsedCallback PreambleCallback) { std::shared_ptr<ASTWorker> Worker(new ASTWorker( - File, Barrier, std::move(AST), /*RunSync=*/!Tasks, UpdateDebounce)); + FileName, IdleASTs, Barrier, /*RunSync=*/!Tasks, UpdateDebounce, + std::move(PCHs), StorePreamblesInMemory, std::move(PreambleCallback))); if (Tasks) - Tasks->runAsync("worker:" + llvm::sys::path::filename(File), + Tasks->runAsync("worker:" + llvm::sys::path::filename(FileName), [Worker]() { Worker->run(); }); return ASTWorkerHandle(std::move(Worker)); } -ASTWorker::ASTWorker(llvm::StringRef File, Semaphore &Barrier, CppFile AST, - bool RunSync, steady_clock::duration UpdateDebounce) - : File(File), RunSync(RunSync), UpdateDebounce(UpdateDebounce), - Barrier(Barrier), AST(std::move(AST)), Done(false) { - if (RunSync) - return; -} +ASTWorker::ASTWorker(PathRef FileName, TUScheduler::ASTCache &LRUCache, + Semaphore &Barrier, bool RunSync, + steady_clock::duration UpdateDebounce, + std::shared_ptr<PCHContainerOperations> PCHs, + bool StorePreamblesInMemory, + PreambleParsedCallback PreambleCallback) + : IdleASTs(LRUCache), RunSync(RunSync), UpdateDebounce(UpdateDebounce), + FileName(FileName), StorePreambleInMemory(StorePreamblesInMemory), + PreambleCallback(std::move(PreambleCallback)), PCHs(std::move(PCHs)), + Barrier(Barrier), Done(false) {} ASTWorker::~ASTWorker() { + // Make sure we remove the cached AST, if any. + IdleASTs.take(this); #ifndef NDEBUG std::lock_guard<std::mutex> Lock(Mutex); assert(Done && "handle was not destroyed"); @@ -213,20 +313,41 @@ void ASTWorker::update(ParseInputs Inputs, WantDiagnostics WantDiags, UniqueFunction<void(std::vector<Diag>)> OnUpdated) { auto Task = [=](decltype(OnUpdated) OnUpdated) mutable { + tooling::CompileCommand OldCommand = std::move(FileInputs.CompileCommand); FileInputs = Inputs; - auto Diags = AST.rebuild(std::move(Inputs)); + // Remove the old AST if it's still in cache. + IdleASTs.take(this); + + log("Updating file " + FileName + " with command [" + + Inputs.CompileCommand.Directory + "] " + + llvm::join(Inputs.CompileCommand.CommandLine, " ")); + // Rebuild the preamble and the AST. + Invocation = buildCompilerInvocation(Inputs); + if (!Invocation) { + log("Could not build CompilerInvocation for file " + FileName); + return; + } + std::shared_ptr<const PreambleData> NewPreamble = buildPreamble( + FileName, *Invocation, getPossiblyStalePreamble(), OldCommand, Inputs, + PCHs, StorePreambleInMemory, PreambleCallback); { std::lock_guard<std::mutex> Lock(Mutex); - if (AST.getPreamble()) - LastBuiltPreamble = AST.getPreamble(); - LastASTSize = AST.getUsedBytes(); + if (NewPreamble) + LastBuiltPreamble = NewPreamble; } + // Build the AST for diagnostics. + llvm::Optional<ParsedAST> AST = + buildAST(FileName, llvm::make_unique<CompilerInvocation>(*Invocation), + Inputs, NewPreamble, PCHs); // We want to report the diagnostics even if this update was cancelled. // It seems more useful than making the clients wait indefinitely if they // spam us with updates. - if (Diags && WantDiags != WantDiagnostics::No) - OnUpdated(std::move(*Diags)); + if (WantDiags != WantDiagnostics::No && AST) + OnUpdated(AST->getDiagnostics()); + // Stash the AST in the cache for further use. + IdleASTs.put(this, + AST ? llvm::make_unique<ParsedAST>(std::move(*AST)) : nullptr); }; startTask("Update", Bind(Task, std::move(OnUpdated)), WantDiags); @@ -236,20 +357,26 @@ llvm::StringRef Name, UniqueFunction<void(llvm::Expected<InputsAndAST>)> Action) { auto Task = [=](decltype(Action) Action) { - ParsedAST *ActualAST = AST.getAST(); - if (!ActualAST) { - Action(llvm::make_error<llvm::StringError>("invalid AST", - llvm::errc::invalid_argument)); - return; + llvm::Optional<std::unique_ptr<ParsedAST>> AST = IdleASTs.take(this); + if (!AST) { + // Try rebuilding the AST. + llvm::Optional<ParsedAST> NewAST = + Invocation + ? buildAST(FileName, + llvm::make_unique<CompilerInvocation>(*Invocation), + FileInputs, getPossiblyStalePreamble(), PCHs) + : llvm::None; + AST = NewAST ? llvm::make_unique<ParsedAST>(std::move(*NewAST)) : nullptr; } - Action(InputsAndAST{FileInputs, *ActualAST}); - - // Size of the AST might have changed after reads too, e.g. if some decls - // were deserialized from preamble. - std::lock_guard<std::mutex> Lock(Mutex); - LastASTSize = ActualAST->getUsedBytes(); + // Make sure we put the AST back into the LRU cache. + auto _ = llvm::make_scope_exit( + [&AST, this]() { IdleASTs.put(this, std::move(*AST)); }); + // Run the user-provided action. + if (!*AST) + return Action(llvm::make_error<llvm::StringError>( + "invalid AST", llvm::errc::invalid_argument)); + Action(InputsAndAST{FileInputs, **AST}); }; - startTask(Name, Bind(Task, std::move(Action)), /*UpdateType=*/llvm::None); } @@ -261,10 +388,17 @@ } std::size_t ASTWorker::getUsedBytes() const { - std::lock_guard<std::mutex> Lock(Mutex); - return LastASTSize; + // Note that we don't report the size of ASTs currently used for processing + // the in-flight requests. We used this information for debugging purposes + // only, so this should be fine. + std::size_t Result = IdleASTs.getUsedBytes(this); + if (auto Preamble = getPossiblyStalePreamble()) + Result += Preamble->Preamble.getSize(); + return Result; } +bool ASTWorker::isASTCached() const { return IdleASTs.getUsedBytes(this) != 0; } + void ASTWorker::stop() { { std::lock_guard<std::mutex> Lock(Mutex); @@ -278,7 +412,7 @@ llvm::Optional<WantDiagnostics> UpdateType) { if (RunSync) { assert(!Done && "running a task after stop()"); - trace::Span Tracer(Name + ":" + llvm::sys::path::filename(File)); + trace::Span Tracer(Name + ":" + llvm::sys::path::filename(FileName)); Task(); return; } @@ -415,10 +549,12 @@ TUScheduler::TUScheduler(unsigned AsyncThreadsCount, bool StorePreamblesInMemory, PreambleParsedCallback PreambleCallback, - steady_clock::duration UpdateDebounce) + std::chrono::steady_clock::duration UpdateDebounce, + ASTRetentionPolicy RetentionPolicy) : StorePreamblesInMemory(StorePreamblesInMemory), PCHOps(std::make_shared<PCHContainerOperations>()), PreambleCallback(std::move(PreambleCallback)), Barrier(AsyncThreadsCount), + IdleASTs(llvm::make_unique<ASTCache>(RetentionPolicy.MaxRetainedASTs)), UpdateDebounce(UpdateDebounce) { if (0 < AsyncThreadsCount) { PreambleTasks.emplace(); @@ -454,9 +590,9 @@ if (!FD) { // Create a new worker to process the AST-related tasks. ASTWorkerHandle Worker = ASTWorker::Create( - File, WorkerThreads ? WorkerThreads.getPointer() : nullptr, Barrier, - CppFile(File, StorePreamblesInMemory, PCHOps, PreambleCallback), - UpdateDebounce); + File, *IdleASTs, WorkerThreads ? WorkerThreads.getPointer() : nullptr, + Barrier, UpdateDebounce, PCHOps, StorePreamblesInMemory, + PreambleCallback); FD = std::unique_ptr<FileData>(new FileData{ Inputs.Contents, Inputs.CompileCommand, std::move(Worker)}); } else { @@ -538,5 +674,15 @@ return Result; } +std::vector<Path> TUScheduler::getFilesWithCachedAST() const { + std::vector<Path> Result; + for (auto &&PathAndFile : Files) { + if (!PathAndFile.second->Worker->isASTCached()) + continue; + Result.push_back(PathAndFile.first()); + } + return Result; +} + } // namespace clangd } // namespace clang Index: clangd/ClangdUnit.h =================================================================== --- clangd/ClangdUnit.h +++ clangd/ClangdUnit.h @@ -47,6 +47,7 @@ PreambleData(PrecompiledPreamble Preamble, std::vector<Diag> Diags, std::vector<Inclusion> Inclusions); + tooling::CompileCommand CompileCommand; PrecompiledPreamble Preamble; std::vector<Diag> Diags; // Processes like code completions and go-to-definitions will need #include @@ -128,50 +129,32 @@ using PreambleParsedCallback = std::function<void( PathRef Path, ASTContext &, std::shared_ptr<clang::Preprocessor>)>; -/// Manages resources, required by clangd. Allows to rebuild file with new -/// contents, and provides AST and Preamble for it. -class CppFile { -public: - CppFile(PathRef FileName, bool StorePreamblesInMemory, - std::shared_ptr<PCHContainerOperations> PCHs, - PreambleParsedCallback PreambleCallback); - - /// Rebuild the AST and the preamble. - /// Returns a list of diagnostics or llvm::None, if an error occured. - llvm::Optional<std::vector<Diag>> rebuild(ParseInputs &&Inputs); - /// Returns the last built preamble. - const std::shared_ptr<const PreambleData> &getPreamble() const; - /// Returns the last built AST. - ParsedAST *getAST() const; - /// Returns an estimated size, in bytes, currently occupied by the AST and the - /// Preamble. - std::size_t getUsedBytes() const; - -private: - /// Build a new preamble for \p Inputs. If the current preamble can be reused, - /// it is returned instead. - /// This method is const to ensure we don't incidentally modify any fields. - std::shared_ptr<const PreambleData> - rebuildPreamble(CompilerInvocation &CI, - const tooling::CompileCommand &Command, - IntrusiveRefCntPtr<vfs::FileSystem> FS, - llvm::MemoryBuffer &ContentsBuffer) const; - - const Path FileName; - const bool StorePreamblesInMemory; - - /// The last CompileCommand used to build AST and Preamble. - tooling::CompileCommand Command; - /// The last parsed AST. - llvm::Optional<ParsedAST> AST; - /// The last built Preamble. - std::shared_ptr<const PreambleData> Preamble; - /// Utility class required by clang - std::shared_ptr<PCHContainerOperations> PCHs; - /// This is called after the file is parsed. This can be nullptr if there is - /// no callback. - PreambleParsedCallback PreambleCallback; -}; +/// Builds compiler invocation that could be used to build AST or preamble. +std::unique_ptr<CompilerInvocation> +buildCompilerInvocation(const ParseInputs &Inputs); + +/// Rebuild the preamble for the new inputs unless the old one can be reused. +/// If \p OldPreamble can be reused, it is returned unchanged. +/// If \p OldPreamble is null, always builds the preamble. +/// If \p PreambleCallback is set, it will be run on top of the AST while +/// building the preamble. Note that if the old preamble was reused, no AST is +/// built and, therefore, the callback will not be executed. +std::shared_ptr<const PreambleData> +buildPreamble(PathRef FileName, CompilerInvocation &CI, + std::shared_ptr<const PreambleData> OldPreamble, + const tooling::CompileCommand &OldCompileCommand, + const ParseInputs &Inputs, + std::shared_ptr<PCHContainerOperations> PCHs, bool StoreInMemory, + PreambleParsedCallback PreambleCallback); + +/// Build an AST from provided user inputs. This function does not check if +/// preamble can be reused, as this function expects that \p Preamble is the +/// result of calling buildPreamble. +llvm::Optional<ParsedAST> +buildAST(PathRef FileName, std::unique_ptr<CompilerInvocation> Invocation, + const ParseInputs &Inputs, + std::shared_ptr<const PreambleData> Preamble, + std::shared_ptr<PCHContainerOperations> PCHs); /// Get the beginning SourceLocation at a specified \p Pos. /// May be invalid if Pos is, or if there's no identifier. Index: clangd/ClangdUnit.cpp =================================================================== --- clangd/ClangdUnit.cpp +++ clangd/ClangdUnit.cpp @@ -175,8 +175,12 @@ ASTDiags.EndSourceFile(); std::vector<const Decl *> ParsedDecls = Action->takeTopLevelDecls(); + std::vector<Diag> Diags = ASTDiags.take(); + // Add diagnostics from the preamble, if any. + if (Preamble) + Diags.insert(Diags.begin(), Preamble->Diags.begin(), Preamble->Diags.end()); return ParsedAST(std::move(Preamble), std::move(Clang), std::move(Action), - std::move(ParsedDecls), ASTDiags.take(), + std::move(ParsedDecls), std::move(Diags), std::move(Inclusions)); } @@ -243,120 +247,57 @@ assert(this->Action); } -CppFile::CppFile(PathRef FileName, bool StorePreamblesInMemory, - std::shared_ptr<PCHContainerOperations> PCHs, - PreambleParsedCallback PreambleCallback) - : FileName(FileName), StorePreamblesInMemory(StorePreamblesInMemory), - PCHs(std::move(PCHs)), PreambleCallback(std::move(PreambleCallback)) { - log("Created CppFile for " + FileName); -} - -llvm::Optional<std::vector<Diag>> CppFile::rebuild(ParseInputs &&Inputs) { - log("Rebuilding file " + FileName + " with command [" + - Inputs.CompileCommand.Directory + "] " + - llvm::join(Inputs.CompileCommand.CommandLine, " ")); - +std::unique_ptr<CompilerInvocation> +clangd::buildCompilerInvocation(const ParseInputs &Inputs) { std::vector<const char *> ArgStrs; for (const auto &S : Inputs.CompileCommand.CommandLine) ArgStrs.push_back(S.c_str()); if (Inputs.FS->setCurrentWorkingDirectory(Inputs.CompileCommand.Directory)) { - log("Couldn't set working directory"); - // We run parsing anyway, our lit-tests rely on results for non-existing - // working dirs. - } - - // Prepare CompilerInvocation. - std::unique_ptr<CompilerInvocation> CI; - { - // FIXME(ibiryukov): store diagnostics from CommandLine when we start - // reporting them. - IgnoreDiagnostics IgnoreDiagnostics; - IntrusiveRefCntPtr<DiagnosticsEngine> CommandLineDiagsEngine = - CompilerInstance::createDiagnostics(new DiagnosticOptions, - &IgnoreDiagnostics, false); - CI = createInvocationFromCommandLine(ArgStrs, CommandLineDiagsEngine, - Inputs.FS); - if (!CI) { - log("Could not build CompilerInvocation for file " + FileName); - AST = llvm::None; - Preamble = nullptr; - return llvm::None; - } - // createInvocationFromCommandLine sets DisableFree. - CI->getFrontendOpts().DisableFree = false; - CI->getLangOpts()->CommentOpts.ParseAllComments = true; + log("Couldn't set working directory when creating compiler invocation."); + // We proceed anyway, our lit-tests rely on results for non-existing working + // dirs. } - std::unique_ptr<llvm::MemoryBuffer> ContentsBuffer = - llvm::MemoryBuffer::getMemBufferCopy(Inputs.Contents, FileName); - - // Compute updated Preamble. - std::shared_ptr<const PreambleData> NewPreamble = - rebuildPreamble(*CI, Inputs.CompileCommand, Inputs.FS, *ContentsBuffer); - - // Remove current AST to avoid wasting memory. - AST = llvm::None; - // Compute updated AST. - llvm::Optional<ParsedAST> NewAST; - { - trace::Span Tracer("Build"); - SPAN_ATTACH(Tracer, "File", FileName); - NewAST = ParsedAST::Build(std::move(CI), NewPreamble, - std::move(ContentsBuffer), PCHs, Inputs.FS); - } - - std::vector<Diag> Diagnostics; - if (NewAST) { - // Collect diagnostics from both the preamble and the AST. - if (NewPreamble) - Diagnostics = NewPreamble->Diags; - Diagnostics.insert(Diagnostics.end(), NewAST->getDiagnostics().begin(), - NewAST->getDiagnostics().end()); - } - - // Write the results of rebuild into class fields. - Command = std::move(Inputs.CompileCommand); - Preamble = std::move(NewPreamble); - AST = std::move(NewAST); - return Diagnostics; -} - -const std::shared_ptr<const PreambleData> &CppFile::getPreamble() const { - return Preamble; -} - -ParsedAST *CppFile::getAST() const { - // We could add mutable to AST instead of const_cast here, but that would also - // allow writing to AST from const methods. - return AST ? const_cast<ParsedAST *>(AST.getPointer()) : nullptr; -} - -std::size_t CppFile::getUsedBytes() const { - std::size_t Total = 0; - if (AST) - Total += AST->getUsedBytes(); - if (StorePreamblesInMemory && Preamble) - Total += Preamble->Preamble.getSize(); - return Total; + // FIXME(ibiryukov): store diagnostics from CommandLine when we start + // reporting them. + IgnoreDiagnostics IgnoreDiagnostics; + IntrusiveRefCntPtr<DiagnosticsEngine> CommandLineDiagsEngine = + CompilerInstance::createDiagnostics(new DiagnosticOptions, + &IgnoreDiagnostics, false); + std::unique_ptr<CompilerInvocation> CI = createInvocationFromCommandLine( + ArgStrs, CommandLineDiagsEngine, Inputs.FS); + if (!CI) + return nullptr; + // createInvocationFromCommandLine sets DisableFree. + CI->getFrontendOpts().DisableFree = false; + CI->getLangOpts()->CommentOpts.ParseAllComments = true; + return CI; } -std::shared_ptr<const PreambleData> -CppFile::rebuildPreamble(CompilerInvocation &CI, - const tooling::CompileCommand &Command, - IntrusiveRefCntPtr<vfs::FileSystem> FS, - llvm::MemoryBuffer &ContentsBuffer) const { - const auto &OldPreamble = this->Preamble; - auto Bounds = ComputePreambleBounds(*CI.getLangOpts(), &ContentsBuffer, 0); - if (OldPreamble && compileCommandsAreEqual(this->Command, Command) && - OldPreamble->Preamble.CanReuse(CI, &ContentsBuffer, Bounds, FS.get())) { +std::shared_ptr<const PreambleData> clangd::buildPreamble( + PathRef FileName, CompilerInvocation &CI, + std::shared_ptr<const PreambleData> OldPreamble, + const tooling::CompileCommand &OldCompileCommand, const ParseInputs &Inputs, + std::shared_ptr<PCHContainerOperations> PCHs, bool StoreInMemory, + PreambleParsedCallback PreambleCallback) { + // Note that we don't need to copy the input contents, preamble can live + // without those. + auto ContentsBuffer = llvm::MemoryBuffer::getMemBuffer(Inputs.Contents); + auto Bounds = + ComputePreambleBounds(*CI.getLangOpts(), ContentsBuffer.get(), 0); + + if (OldPreamble && + compileCommandsAreEqual(Inputs.CompileCommand, OldCompileCommand) && + OldPreamble->Preamble.CanReuse(CI, ContentsBuffer.get(), Bounds, + Inputs.FS.get())) { log("Reusing preamble for file " + Twine(FileName)); return OldPreamble; } log("Preamble for file " + Twine(FileName) + " cannot be reused. Attempting to rebuild it."); - trace::Span Tracer("Preamble"); + trace::Span Tracer("BuildPreamble"); SPAN_ATTACH(Tracer, "File", FileName); StoreDiags PreambleDiagnostics; IntrusiveRefCntPtr<DiagnosticsEngine> PreambleDiagsEngine = @@ -369,18 +310,22 @@ CI.getFrontendOpts().SkipFunctionBodies = true; CppFilePreambleCallbacks SerializedDeclsCollector(FileName, PreambleCallback); + if (Inputs.FS->setCurrentWorkingDirectory(Inputs.CompileCommand.Directory)) { + log("Couldn't set working directory when building the preamble."); + // We proceed anyway, our lit-tests rely on results for non-existing working + // dirs. + } auto BuiltPreamble = PrecompiledPreamble::Build( - CI, &ContentsBuffer, Bounds, *PreambleDiagsEngine, FS, PCHs, - /*StoreInMemory=*/StorePreamblesInMemory, SerializedDeclsCollector); + CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine, Inputs.FS, PCHs, + StoreInMemory, SerializedDeclsCollector); // When building the AST for the main file, we do want the function // bodies. CI.getFrontendOpts().SkipFunctionBodies = false; if (BuiltPreamble) { log("Built preamble of size " + Twine(BuiltPreamble->getSize()) + " for file " + Twine(FileName)); - return std::make_shared<PreambleData>( std::move(*BuiltPreamble), PreambleDiagnostics.take(), SerializedDeclsCollector.takeInclusions()); @@ -390,6 +335,24 @@ } } +llvm::Optional<ParsedAST> clangd::buildAST( + PathRef FileName, std::unique_ptr<CompilerInvocation> Invocation, + const ParseInputs &Inputs, std::shared_ptr<const PreambleData> Preamble, + std::shared_ptr<PCHContainerOperations> PCHs) { + trace::Span Tracer("BuildAST"); + SPAN_ATTACH(Tracer, "File", FileName); + + if (Inputs.FS->setCurrentWorkingDirectory(Inputs.CompileCommand.Directory)) { + log("Couldn't set working directory when building the preamble."); + // We proceed anyway, our lit-tests rely on results for non-existing working + // dirs. + } + + return ParsedAST::Build( + llvm::make_unique<CompilerInvocation>(*Invocation), Preamble, + llvm::MemoryBuffer::getMemBufferCopy(Inputs.Contents), PCHs, Inputs.FS); +} + SourceLocation clangd::getBeginningOfIdentifier(ParsedAST &Unit, const Position &Pos, const FileID FID) { Index: clangd/ClangdServer.h =================================================================== --- clangd/ClangdServer.h +++ clangd/ClangdServer.h @@ -70,6 +70,9 @@ /// If 0, all requests are processed on the calling thread. unsigned AsyncThreadsCount = getDefaultAsyncThreadsCount(); + /// AST caching policy. The default is to keep up to 3 ASTs in memory. + ASTRetentionPolicy RetentionPolicy; + /// Cached preambles are potentially large. If false, store them on disk. bool StorePreamblesInMemory = true; Index: clangd/ClangdServer.cpp =================================================================== --- clangd/ClangdServer.cpp +++ clangd/ClangdServer.cpp @@ -100,7 +100,8 @@ std::shared_ptr<Preprocessor> PP) { FileIdx->update(Path, &AST, std::move(PP)); } : PreambleParsedCallback(), - Opts.UpdateDebounce) { + Opts.UpdateDebounce, + Opts.RetentionPolicy) { if (FileIdx && Opts.StaticIndex) { MergedIndex = mergeIndex(FileIdx.get(), Opts.StaticIndex); Index = MergedIndex.get();
_______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits