kadircet updated this revision to Diff 210564.
kadircet added a comment.
- Add comments
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D64712/new/
https://reviews.llvm.org/D64712
Files:
clang-tools-extra/clangd/CMakeLists.txt
clang-tools-extra/clangd/index/Background.cpp
clang-tools-extra/clangd/index/Background.h
clang-tools-extra/clangd/index/BackgroundIndexLoader.cpp
clang-tools-extra/clangd/index/BackgroundIndexLoader.h
clang-tools-extra/clangd/index/BackgroundRebuild.cpp
clang-tools-extra/clangd/index/BackgroundRebuild.h
clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
Index: clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
===================================================================
--- clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
+++ clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
@@ -211,7 +211,7 @@
OverlayCDB CDB(/*Base=*/nullptr);
BackgroundIndex Idx(Context::empty(), FS, CDB,
[&](llvm::StringRef) { return &MSS; });
- CDB.setCompileCommand(testPath("root"), Cmd);
+ CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
ASSERT_TRUE(Idx.blockUntilIdleForTest());
}
EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache.
@@ -335,7 +335,7 @@
OverlayCDB CDB(/*Base=*/nullptr);
BackgroundIndex Idx(Context::empty(), FS, CDB,
[&](llvm::StringRef) { return &MSS; });
- CDB.setCompileCommand(testPath("root"), Cmd);
+ CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
ASSERT_TRUE(Idx.blockUntilIdleForTest());
}
EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache.
@@ -353,7 +353,7 @@
OverlayCDB CDB(/*Base=*/nullptr);
BackgroundIndex Idx(Context::empty(), FS, CDB,
[&](llvm::StringRef) { return &MSS; });
- CDB.setCompileCommand(testPath("root"), Cmd);
+ CDB.setCompileCommand(testPath("root/A.cc"), Cmd);
ASSERT_TRUE(Idx.blockUntilIdleForTest());
}
EXPECT_EQ(CacheHits, 2U); // Check both A.cc and A.h loaded from cache.
@@ -621,8 +621,8 @@
TEST_F(BackgroundIndexRebuilderTest, LoadingShards) {
Rebuilder.startLoading();
- Rebuilder.loadedTU();
- Rebuilder.loadedTU();
+ Rebuilder.loadedShard(10);
+ Rebuilder.loadedShard(20);
EXPECT_TRUE(checkRebuild([&] { Rebuilder.doneLoading(); }));
// No rebuild for no shards.
@@ -631,11 +631,11 @@
// Loads can overlap.
Rebuilder.startLoading();
- Rebuilder.loadedTU();
+ Rebuilder.loadedShard(1);
Rebuilder.startLoading();
- Rebuilder.loadedTU();
+ Rebuilder.loadedShard(1);
EXPECT_FALSE(checkRebuild([&] { Rebuilder.doneLoading(); }));
- Rebuilder.loadedTU();
+ Rebuilder.loadedShard(1);
EXPECT_TRUE(checkRebuild([&] { Rebuilder.doneLoading(); }));
// No rebuilding for indexed files while loading.
Index: clang-tools-extra/clangd/index/BackgroundRebuild.h
===================================================================
--- clang-tools-extra/clangd/index/BackgroundRebuild.h
+++ clang-tools-extra/clangd/index/BackgroundRebuild.h
@@ -16,6 +16,7 @@
#include "index/FileIndex.h"
#include "index/Index.h"
+#include <cstddef>
namespace clang {
namespace clangd {
@@ -63,7 +64,7 @@
// sessions may happen concurrently.
void startLoading();
// Called to indicate some shards were actually loaded from disk.
- void loadedTU();
+ void loadedShard(size_t ShardCount);
// Called to indicate we're finished loading shards from disk.
// May rebuild (if any were loaded).
void doneLoading();
@@ -87,7 +88,7 @@
unsigned IndexedTUsAtLastRebuild = 0;
// Are we loading shards? May be multiple concurrent sessions.
unsigned Loading = 0;
- unsigned LoadedTUs; // In the current loading session.
+ unsigned LoadedShards; // In the current loading session.
SwapIndex *Target;
FileSymbols *Source;
Index: clang-tools-extra/clangd/index/BackgroundRebuild.cpp
===================================================================
--- clang-tools-extra/clangd/index/BackgroundRebuild.cpp
+++ clang-tools-extra/clangd/index/BackgroundRebuild.cpp
@@ -78,13 +78,13 @@
void BackgroundIndexRebuilder::startLoading() {
std::lock_guard<std::mutex> Lock(Mu);
if (!Loading)
- LoadedTUs = 0;
+ LoadedShards = 0;
++Loading;
}
-void BackgroundIndexRebuilder::loadedTU() {
+void BackgroundIndexRebuilder::loadedShard(size_t ShardCount) {
std::lock_guard<std::mutex> Lock(Mu);
assert(Loading);
- ++LoadedTUs;
+ LoadedShards += ShardCount;
}
void BackgroundIndexRebuilder::doneLoading() {
maybeRebuild("after loading index from disk", [this] {
@@ -93,7 +93,7 @@
if (Loading) // was loading multiple batches concurrently
return false; // rebuild once the last batch is done.
// Rebuild if we loaded any shards, or if we stopped an indexedTU rebuild.
- return LoadedTUs > 0 || enoughTUsToRebuild();
+ return LoadedShards > 0 || enoughTUsToRebuild();
});
}
Index: clang-tools-extra/clangd/index/BackgroundIndexLoader.h
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/index/BackgroundIndexLoader.h
@@ -0,0 +1,54 @@
+//===--- BackgroundIndexLoader.h - Load shards from index storage-*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_BACKGROUND_INDEX_LOADER_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_BACKGROUND_INDEX_LOADER_H
+
+#include "Path.h"
+#include "index/Background.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/VirtualFileSystem.h"
+#include <memory>
+#include <vector>
+
+namespace clang {
+namespace clangd {
+
+/// Represents a shard loaded from storage, stores contents in \p Shard and
+/// metadata about the source file that generated this shard.
+struct LoadedShard {
+ /// Path of the source file that produced this shard.
+ Path AbsolutePath;
+ /// Digest of the source file contents that produced this shard.
+ FileDigest Digest = {};
+ /// Whether the RefSlab in Shard should be used for updating symbol reference
+ /// counts when building an index.
+ bool CountReferences = false;
+ /// Whether the indexing action producing that shard had errors.
+ bool HadErrors = false;
+ /// Path to a TU that is depending on this shard.
+ Path DependentTU;
+ /// Will be nullptr when index storage couldn't provide a valid shard for
+ /// AbsolutePath.
+ std::unique_ptr<IndexFileIn> Shard;
+};
+
+/// Loads all shards for the TU \p MainFile from \p Storage.
+std::vector<LoadedShard>
+loadIndexShards(llvm::ArrayRef<Path> MainFiles,
+ BackgroundIndexStorage::Factory &IndexStorageFactory,
+ const GlobalCompilationDatabase &CDB);
+
+} // namespace clangd
+} // namespace clang
+
+#endif
Index: clang-tools-extra/clangd/index/BackgroundIndexLoader.cpp
===================================================================
--- /dev/null
+++ clang-tools-extra/clangd/index/BackgroundIndexLoader.cpp
@@ -0,0 +1,162 @@
+//===-- BackgroundIndexLoader.cpp - ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "index/BackgroundIndexLoader.h"
+#include "GlobalCompilationDatabase.h"
+#include "Logger.h"
+#include "Path.h"
+#include "index/Background.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/Support/Path.h"
+#include <string>
+#include <vector>
+
+namespace clang {
+namespace clangd {
+namespace {
+
+llvm::Optional<Path> uriToAbsolutePath(llvm::StringRef URI, PathRef HintPath) {
+ auto U = URI::parse(URI);
+ if (!U)
+ return llvm::None;
+ auto AbsolutePath = URI::resolve(*U, HintPath);
+ if (!AbsolutePath)
+ return llvm::None;
+ return *AbsolutePath;
+}
+
+/// A helper class to cache BackgroundIndexStorage operations and keep the
+/// inverse dependency mapping.
+class BackgroundIndexLoader {
+public:
+ /// Load the shards for \p MainFile and all of its dependencies.
+ void load(PathRef MainFile, BackgroundIndexStorage *Storage);
+
+ /// Consumes the loader and returns all shards.
+ std::vector<LoadedShard> takeResult() &&;
+
+private:
+ struct CachedShard {
+ LoadedShard LS;
+ // Stores the absolute paths for the direct dependencies of this file.
+ // FIXME: This information is also available in LS but in URI format, rather
+ // make use of a URIToFileCache and get rid of this field.
+ std::vector<Path> Edges;
+ };
+ /// Returns the Shard for \p StartSourceFile from cache or loads it from \p
+ /// Storage.
+ const CachedShard &loadShard(PathRef StartSourceFile,
+ BackgroundIndexStorage *Storage);
+
+ /// Cache for Storage lookups.
+ llvm::StringMap<CachedShard> LoadedShards;
+
+ /// References are into the AbsolutePaths in LoadedShards.
+ llvm::DenseMap<PathRef, PathRef> FileToTU;
+};
+
+const BackgroundIndexLoader::CachedShard &
+BackgroundIndexLoader::loadShard(PathRef StartSourceFile,
+ BackgroundIndexStorage *Storage) {
+ auto It = LoadedShards.try_emplace(StartSourceFile);
+ CachedShard &CS = It.first->getValue();
+ // Return the cached shard.
+ if (!It.second)
+ return CS;
+
+ LoadedShard &LS = CS.LS;
+ LS.AbsolutePath = StartSourceFile.str();
+ auto Shard = Storage->loadShard(StartSourceFile);
+ if (!Shard || !Shard->Sources) {
+ vlog("Failed to load shard: {0}", StartSourceFile);
+ LS.HadErrors = true;
+ return CS;
+ }
+
+ LS.Shard = std::move(Shard);
+ for (const auto &It : *LS.Shard->Sources) {
+ auto AbsPath = uriToAbsolutePath(It.getKey(), StartSourceFile);
+ if (!AbsPath)
+ continue;
+ // A shard contains only edges for non main-file sources.
+ if (*AbsPath != StartSourceFile) {
+ CS.Edges.push_back(*AbsPath);
+ continue;
+ }
+
+ // Fill in shard metadata.
+ const IncludeGraphNode &IGN = It.getValue();
+ LS.Digest = IGN.Digest;
+ LS.CountReferences = IGN.Flags & IncludeGraphNode::SourceFlag::IsTU;
+ LS.HadErrors = IGN.Flags & IncludeGraphNode::SourceFlag::HadErrors;
+ }
+ assert(LS.Digest != FileDigest{{0}} && "Digest is empty?");
+ return CS;
+}
+
+void BackgroundIndexLoader::load(PathRef MainFile,
+ BackgroundIndexStorage *Storage) {
+ // Following containers points to strings inside LoadedShards.
+ llvm::DenseSet<PathRef> InQueue;
+ std::queue<PathRef> ToVisit;
+
+ const CachedShard &MainShard = loadShard(MainFile, Storage);
+ ToVisit.push(MainShard.LS.AbsolutePath);
+ InQueue.insert(MainShard.LS.AbsolutePath);
+
+ while (!ToVisit.empty()) {
+ PathRef SourceFile = ToVisit.front();
+ ToVisit.pop();
+
+ const CachedShard &CS = loadShard(SourceFile, Storage);
+ FileToTU.try_emplace(CS.LS.AbsolutePath).first->second =
+ MainShard.LS.AbsolutePath;
+
+ for (PathRef Edge : CS.Edges) {
+ if (InQueue.insert(Edge).second)
+ ToVisit.push(Edge);
+ }
+ }
+}
+
+std::vector<LoadedShard> BackgroundIndexLoader::takeResult() && {
+ std::vector<LoadedShard> Result;
+ Result.reserve(LoadedShards.size());
+ for (auto &It : LoadedShards) {
+ Result.push_back(std::move(It.getValue().LS));
+ LoadedShard &LS = Result.back();
+ auto TUsIt = FileToTU.find(LS.AbsolutePath);
+ assert(TUsIt != FileToTU.end() && "No TU registered for the shard");
+ Result.back().DependentTU = TUsIt->second;
+ }
+ return Result;
+}
+} // namespace
+
+std::vector<LoadedShard>
+loadIndexShards(llvm::ArrayRef<Path> MainFiles,
+ BackgroundIndexStorage::Factory &IndexStorageFactory,
+ const GlobalCompilationDatabase &CDB) {
+ BackgroundIndexLoader Loader;
+ for (llvm::StringRef MainFile : MainFiles) {
+ assert(llvm::sys::path::is_absolute(MainFile));
+
+ std::string ProjectRoot;
+ if (auto PI = CDB.getProjectInfo(MainFile))
+ ProjectRoot = std::move(PI->SourceRoot);
+ BackgroundIndexStorage *Storage = IndexStorageFactory(ProjectRoot);
+ Loader.load(MainFile, Storage);
+ }
+ return std::move(Loader).takeResult();
+}
+
+} // namespace clangd
+} // namespace clang
Index: clang-tools-extra/clangd/index/Background.h
===================================================================
--- clang-tools-extra/clangd/index/Background.h
+++ clang-tools-extra/clangd/index/Background.h
@@ -12,6 +12,7 @@
#include "Context.h"
#include "FSProvider.h"
#include "GlobalCompilationDatabase.h"
+#include "Path.h"
#include "SourceCode.h"
#include "Threading.h"
#include "index/BackgroundRebuild.h"
@@ -173,20 +174,9 @@
std::mutex ShardVersionsMu;
BackgroundIndexStorage::Factory IndexStorageFactory;
- struct Source {
- std::string Path;
- bool NeedsReIndexing;
- Source(llvm::StringRef Path, bool NeedsReIndexing)
- : Path(Path), NeedsReIndexing(NeedsReIndexing) {}
- };
- // Loads the shards for a single TU and all of its dependencies. Returns the
- // list of sources and whether they need to be re-indexed.
- std::vector<Source> loadShard(const tooling::CompileCommand &Cmd,
- BackgroundIndexStorage *IndexStorage,
- llvm::StringSet<> &LoadedShards);
- // Tries to load shards for the ChangedFiles.
+ // Tries to load shards for the MainFiles and their dependencies.
std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
- loadShards(std::vector<std::string> ChangedFiles);
+ loadProject(std::vector<std::string> MainFiles);
BackgroundQueue::Task
changedFilesTask(const std::vector<std::string> &ChangedFiles);
Index: clang-tools-extra/clangd/index/Background.cpp
===================================================================
--- clang-tools-extra/clangd/index/Background.cpp
+++ clang-tools-extra/clangd/index/Background.cpp
@@ -10,6 +10,7 @@
#include "ClangdUnit.h"
#include "Compiler.h"
#include "Context.h"
+#include "FSProvider.h"
#include "Headers.h"
#include "Logger.h"
#include "Path.h"
@@ -18,6 +19,7 @@
#include "Threading.h"
#include "Trace.h"
#include "URI.h"
+#include "index/BackgroundIndexLoader.h"
#include "index/FileIndex.h"
#include "index/IndexAction.h"
#include "index/MemIndex.h"
@@ -28,6 +30,8 @@
#include "clang/Basic/SourceLocation.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Driver/Types.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
@@ -42,6 +46,7 @@
#include <atomic>
#include <chrono>
#include <condition_variable>
+#include <cstddef>
#include <memory>
#include <mutex>
#include <numeric>
@@ -49,6 +54,8 @@
#include <random>
#include <string>
#include <thread>
+#include <utility>
+#include <vector>
namespace clang {
namespace clangd {
@@ -119,6 +126,18 @@
}
return AbsolutePath;
}
+
+bool shardIsStale(const LoadedShard &LS, llvm::vfs::FileSystem *FS) {
+ auto Buf = FS->getBufferForFile(LS.AbsolutePath);
+ if (!Buf) {
+ elog("Background-index: Couldn't read {0} to validate stored index: {1}",
+ LS.AbsolutePath, Buf.getError().message());
+ // There is no point in indexing an unreadable file.
+ return false;
+ }
+ return digest(Buf->get()->getBuffer()) != LS.Digest;
+}
+
} // namespace
BackgroundIndex::BackgroundIndex(
@@ -156,7 +175,7 @@
log("Enqueueing {0} commands for indexing", ChangedFiles.size());
SPAN_ATTACH(Tracer, "files", int64_t(ChangedFiles.size()));
- auto NeedsReIndexing = loadShards(std::move(ChangedFiles));
+ auto NeedsReIndexing = loadProject(std::move(ChangedFiles));
// Run indexing for files that need to be updated.
std::shuffle(NeedsReIndexing.begin(), NeedsReIndexing.end(),
std::mt19937(std::random_device{}()));
@@ -431,169 +450,77 @@
return llvm::Error::success();
}
-std::vector<BackgroundIndex::Source>
-BackgroundIndex::loadShard(const tooling::CompileCommand &Cmd,
- BackgroundIndexStorage *IndexStorage,
- llvm::StringSet<> &LoadedShards) {
- struct ShardInfo {
- std::string AbsolutePath;
- std::unique_ptr<IndexFileIn> Shard;
- FileDigest Digest = {};
- bool CountReferences = false;
- bool HadErrors = false;
- };
- std::vector<ShardInfo> IntermediateSymbols;
- // Make sure we don't have duplicate elements in the queue. Keys are absolute
- // paths.
- llvm::StringSet<> InQueue;
- auto FS = FSProvider.getFileSystem();
- // Dependencies of this TU, paired with the information about whether they
- // need to be re-indexed or not.
- std::vector<Source> Dependencies;
- std::queue<Source> ToVisit;
- std::string AbsolutePath = getAbsolutePath(Cmd).str();
- // Up until we load the shard related to a dependency it needs to be
- // re-indexed.
- ToVisit.emplace(AbsolutePath, true);
- InQueue.insert(AbsolutePath);
- // Goes over each dependency.
- while (!ToVisit.empty()) {
- Dependencies.push_back(std::move(ToVisit.front()));
- // Dependencies is not modified during the rest of the loop, so it is safe
- // to keep the reference.
- auto &CurDependency = Dependencies.back();
- ToVisit.pop();
- // If we have already seen this shard before(either loaded or failed) don't
- // re-try again. Since the information in the shard won't change from one TU
- // to another.
- if (!LoadedShards.try_emplace(CurDependency.Path).second) {
- // If the dependency needs to be re-indexed, first occurence would already
- // have detected that, so we don't need to issue it again.
- CurDependency.NeedsReIndexing = false;
- continue;
- }
-
- auto Shard = IndexStorage->loadShard(CurDependency.Path);
- if (!Shard || !Shard->Sources) {
- // File will be returned as requiring re-indexing to caller.
- vlog("Failed to load shard: {0}", CurDependency.Path);
- continue;
- }
- // These are the edges in the include graph for current dependency.
- for (const auto &I : *Shard->Sources) {
- auto U = URI::parse(I.getKey());
- if (!U)
- continue;
- auto AbsolutePath = URI::resolve(*U, CurDependency.Path);
- if (!AbsolutePath)
- continue;
- // Add file as dependency if haven't seen before.
- if (InQueue.try_emplace(*AbsolutePath).second)
- ToVisit.emplace(*AbsolutePath, true);
- // The node contains symbol information only for current file, the rest is
- // just edges.
- if (*AbsolutePath != CurDependency.Path)
- continue;
+// Restores shards for \p MainFiles from index storage. Then checks staleness of
+// those shards and returns a list of TUs that needs to be indexed to update
+// staleness.
+std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
+BackgroundIndex::loadProject(std::vector<std::string> MainFiles) {
+ std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
+ NeedsReIndexing;
- // We found source file info for current dependency.
- assert(I.getValue().Digest != FileDigest{{0}} && "Digest is empty?");
- ShardInfo SI;
- SI.AbsolutePath = CurDependency.Path;
- SI.Shard = std::move(Shard);
- SI.Digest = I.getValue().Digest;
- SI.CountReferences =
- I.getValue().Flags & IncludeGraphNode::SourceFlag::IsTU;
- SI.HadErrors =
- I.getValue().Flags & IncludeGraphNode::SourceFlag::HadErrors;
- IntermediateSymbols.push_back(std::move(SI));
- // Check if the source needs re-indexing.
- // Get the digest, skip it if file doesn't exist.
- auto Buf = FS->getBufferForFile(CurDependency.Path);
- if (!Buf) {
- elog("Couldn't get buffer for file: {0}: {1}", CurDependency.Path,
- Buf.getError().message());
- continue;
- }
- // If digests match then dependency doesn't need re-indexing.
- // FIXME: Also check for dependencies(sources) of this shard and compile
- // commands for cache invalidation.
- CurDependency.NeedsReIndexing =
- digest(Buf->get()->getBuffer()) != I.getValue().Digest;
- }
- }
- // Load shard information into background-index.
+ Rebuilder.startLoading();
+ // Load shards for all of the mainfiles.
+ const std::vector<LoadedShard> Result =
+ loadIndexShards(MainFiles, IndexStorageFactory, CDB);
+ size_t LoadedShards = 0;
{
+ // Update in-memory state.
std::lock_guard<std::mutex> Lock(ShardVersionsMu);
- // This can override a newer version that is added in another thread,
- // if this thread sees the older version but finishes later. This
- // should be rare in practice.
- for (const ShardInfo &SI : IntermediateSymbols) {
+ for (auto &LS : Result) {
+ if (!LS.Shard)
+ continue;
auto SS =
- SI.Shard->Symbols
- ? llvm::make_unique<SymbolSlab>(std::move(*SI.Shard->Symbols))
+ LS.Shard->Symbols
+ ? llvm::make_unique<SymbolSlab>(std::move(*LS.Shard->Symbols))
: nullptr;
- auto RS = SI.Shard->Refs
- ? llvm::make_unique<RefSlab>(std::move(*SI.Shard->Refs))
+ auto RS = LS.Shard->Refs
+ ? llvm::make_unique<RefSlab>(std::move(*LS.Shard->Refs))
: nullptr;
auto RelS =
- SI.Shard->Relations
- ? llvm::make_unique<RelationSlab>(std::move(*SI.Shard->Relations))
+ LS.Shard->Relations
+ ? llvm::make_unique<RelationSlab>(std::move(*LS.Shard->Relations))
: nullptr;
- ShardVersion &SV = ShardVersions[SI.AbsolutePath];
- SV.Digest = SI.Digest;
- SV.HadErrors = SI.HadErrors;
+ ShardVersion &SV = ShardVersions[LS.AbsolutePath];
+ SV.Digest = LS.Digest;
+ SV.HadErrors = LS.HadErrors;
+ ++LoadedShards;
- IndexedSymbols.update(SI.AbsolutePath, std::move(SS), std::move(RS),
- std::move(RelS), SI.CountReferences);
+ IndexedSymbols.update(LS.AbsolutePath, std::move(SS), std::move(RS),
+ std::move(RelS), LS.CountReferences);
}
}
- if (!IntermediateSymbols.empty())
- Rebuilder.loadedTU();
+ Rebuilder.loadedShard(LoadedShards);
+ Rebuilder.doneLoading();
- return Dependencies;
-}
+ auto FS = FSProvider.getFileSystem();
+ llvm::DenseSet<PathRef> TUsToIndex;
+ // We'll accept data from stale shards, but ensure the files get reindexed
+ // soon.
+ for (auto &LS : Result) {
+ if (!shardIsStale(LS, FS.get()))
+ continue;
+ PathRef TUForFile = LS.DependentTU;
+ assert(!TUForFile.empty() && "File without a TU!");
+
+ // FIXME: Currently, we simply schedule indexing on a TU whenever any of
+ // its dependencies needs re-indexing. We might do it smarter by figuring
+ // out a minimal set of TUs that will cover all the stale dependencies.
+ // FIXME: Try looking at other TUs if no compile commands are available
+ // for this TU, i.e TU was deleted after we performed indexing.
+ TUsToIndex.insert(TUForFile);
+ }
-// Goes over each changed file and loads them from index. Returns the list of
-// TUs that had out-of-date/no shards.
-std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
-BackgroundIndex::loadShards(std::vector<std::string> ChangedFiles) {
- std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
- NeedsReIndexing;
- // Keeps track of the files that will be reindexed, to make sure we won't
- // re-index same dependencies more than once. Keys are AbsolutePaths.
- llvm::StringSet<> FilesToIndex;
- // Keeps track of the loaded shards to make sure we don't perform redundant
- // disk IO. Keys are absolute paths.
- llvm::StringSet<> LoadedShards;
- Rebuilder.startLoading();
- for (const auto &File : ChangedFiles) {
- auto Cmd = CDB.getCompileCommand(File);
+ for (PathRef TU : TUsToIndex) {
+ auto Cmd = CDB.getCompileCommand(TU);
if (!Cmd)
continue;
-
std::string ProjectRoot;
- if (auto PI = CDB.getProjectInfo(File))
+ if (auto PI = CDB.getProjectInfo(TU))
ProjectRoot = std::move(PI->SourceRoot);
-
- BackgroundIndexStorage *IndexStorage = IndexStorageFactory(ProjectRoot);
- auto Dependencies = loadShard(*Cmd, IndexStorage, LoadedShards);
- for (const auto &Dependency : Dependencies) {
- if (!Dependency.NeedsReIndexing || FilesToIndex.count(Dependency.Path))
- continue;
- // FIXME: Currently, we simply schedule indexing on a TU whenever any of
- // its dependencies needs re-indexing. We might do it smarter by figuring
- // out a minimal set of TUs that will cover all the stale dependencies.
- vlog("Enqueueing TU {0} because its dependency {1} needs re-indexing.",
- Cmd->Filename, Dependency.Path);
- NeedsReIndexing.push_back({std::move(*Cmd), IndexStorage});
- // Mark all of this TU's dependencies as to-be-indexed so that we won't
- // try to re-index those.
- for (const auto &Dependency : Dependencies)
- FilesToIndex.insert(Dependency.Path);
- break;
- }
+ BackgroundIndexStorage *Storage = IndexStorageFactory(ProjectRoot);
+ NeedsReIndexing.emplace_back(std::move(*Cmd), Storage);
}
- Rebuilder.doneLoading();
+
return NeedsReIndexing;
}
Index: clang-tools-extra/clangd/CMakeLists.txt
===================================================================
--- clang-tools-extra/clangd/CMakeLists.txt
+++ clang-tools-extra/clangd/CMakeLists.txt
@@ -73,6 +73,7 @@
XRefs.cpp
index/Background.cpp
+ index/BackgroundIndexLoader.cpp
index/BackgroundIndexStorage.cpp
index/BackgroundQueue.cpp
index/BackgroundRebuild.cpp
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
