[PATCH] D90750: [clangd] Introduce ProjectAwareIndex

[Kadir Cetinkaya via Phabricator via cfe-commits]

This revision was landed with ongoing or failed builds.
This revision was automatically updated to reflect the committed changes.
Closed by commit rG067ffbfe6018: [clangd] Introduce ProjectAwareIndex (authored 
by kadircet).

Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D90750/new/

https://reviews.llvm.org/D90750

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/Config.h
  clang-tools-extra/clangd/index/ProjectAware.cpp
  clang-tools-extra/clangd/index/ProjectAware.h
  clang-tools-extra/clangd/unittests/CMakeLists.txt
  clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp

Index: clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp
===
--- /dev/null
+++ clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp
@@ -0,0 +1,86 @@
+//===-- ProjectAwareIndexTests.cpp  ---*- C++ -*---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+
+#include "Config.h"
+#include "TestIndex.h"
+#include "index/Index.h"
+#include "index/MemIndex.h"
+#include "index/ProjectAware.h"
+#include "index/Ref.h"
+#include "index/Relation.h"
+#include "support/Context.h"
+#include "support/Threading.h"
+#include "llvm/ADT/StringRef.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include 
+#include 
+
+namespace clang {
+namespace clangd {
+using testing::ElementsAre;
+using testing::IsEmpty;
+
+std::unique_ptr createIndex() {
+  std::vector Symbols = {symbol("1")};
+  return std::make_unique(std::move(Symbols), RefSlab(),
+RelationSlab());
+}
+
+TEST(ProjectAware, Test) {
+  IndexFactory Gen = [](const Config::ExternalIndexSpec &, AsyncTaskRunner &) {
+return createIndex();
+  };
+
+  auto Idx = createProjectAwareIndex(std::move(Gen));
+  FuzzyFindRequest Req;
+  Req.Query = "1";
+  Req.AnyScope = true;
+
+  EXPECT_THAT(match(*Idx, Req), IsEmpty());
+
+  Config C;
+  C.Index.External.emplace();
+  C.Index.External->Location = "test";
+  WithContextValue With(Config::Key, std::move(C));
+  EXPECT_THAT(match(*Idx, Req), ElementsAre("1"));
+  return;
+}
+
+TEST(ProjectAware, CreatedOnce) {
+  unsigned InvocationCount = 0;
+  IndexFactory Gen = [&](const Config::ExternalIndexSpec &, AsyncTaskRunner &) {
+++InvocationCount;
+return createIndex();
+  };
+
+  auto Idx = createProjectAwareIndex(std::move(Gen));
+  // No invocation at start.
+  EXPECT_EQ(InvocationCount, 0U);
+  FuzzyFindRequest Req;
+  Req.Query = "1";
+  Req.AnyScope = true;
+
+  // Cannot invoke without proper config.
+  match(*Idx, Req);
+  EXPECT_EQ(InvocationCount, 0U);
+
+  Config C;
+  C.Index.External.emplace();
+  C.Index.External->Location = "test";
+  WithContextValue With(Config::Key, std::move(C));
+  match(*Idx, Req);
+  // Now it should be created.
+  EXPECT_EQ(InvocationCount, 1U);
+  match(*Idx, Req);
+  // It is cached afterwards.
+  EXPECT_EQ(InvocationCount, 1U);
+  return;
+}
+} // namespace clangd
+} // namespace clang
Index: clang-tools-extra/clangd/unittests/CMakeLists.txt
===
--- clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -74,6 +74,7 @@
   PathMappingTests.cpp
   PreambleTests.cpp
   PrintASTTests.cpp
+  ProjectAwareIndexTests.cpp
   QualityTests.cpp
   RenameTests.cpp
   RIFFTests.cpp
Index: clang-tools-extra/clangd/index/ProjectAware.h
===
--- /dev/null
+++ clang-tools-extra/clangd/index/ProjectAware.h
@@ -0,0 +1,34 @@
+//===--- ProjectAware.h --*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_PROJECT_AWARE_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_PROJECT_AWARE_H
+
+#include "Config.h"
+#include "index/Index.h"
+#include "support/Threading.h"
+#include 
+#include 
+namespace clang {
+namespace clangd {
+
+/// A functor to create an index for an external index specification. Functor
+/// should perform any high latency operation in a separate thread through
+/// AsyncTaskRunner.
+using IndexFactory = std::function(
+const Config::ExternalIndexSpec &, AsyncTaskRunner &)>;
+
+/// Returns an index that answers queries using external indices. IndexGenerator
+/// can be used to customize how to generate an index from an external source.

[PATCH] D90750: [clangd] Introduce ProjectAwareIndex

[Kadir Cetinkaya via Phabricator via cfe-commits]

kadircet updated this revision to Diff 306924.
kadircet marked an inline comment as done.
kadircet added a comment.

- Address comments


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D90750/new/

https://reviews.llvm.org/D90750

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/Config.h
  clang-tools-extra/clangd/index/ProjectAware.cpp
  clang-tools-extra/clangd/index/ProjectAware.h
  clang-tools-extra/clangd/unittests/CMakeLists.txt
  clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp

Index: clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp
===
--- /dev/null
+++ clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp
@@ -0,0 +1,86 @@
+//===-- ProjectAwareIndexTests.cpp  ---*- C++ -*---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+
+#include "Config.h"
+#include "TestIndex.h"
+#include "index/Index.h"
+#include "index/MemIndex.h"
+#include "index/ProjectAware.h"
+#include "index/Ref.h"
+#include "index/Relation.h"
+#include "support/Context.h"
+#include "support/Threading.h"
+#include "llvm/ADT/StringRef.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include 
+#include 
+
+namespace clang {
+namespace clangd {
+using testing::ElementsAre;
+using testing::IsEmpty;
+
+std::unique_ptr createIndex() {
+  std::vector Symbols = {symbol("1")};
+  return std::make_unique(std::move(Symbols), RefSlab(),
+RelationSlab());
+}
+
+TEST(ProjectAware, Test) {
+  IndexFactory Gen = [](const Config::ExternalIndexSpec &, AsyncTaskRunner &) {
+return createIndex();
+  };
+
+  auto Idx = createProjectAwareIndex(std::move(Gen));
+  FuzzyFindRequest Req;
+  Req.Query = "1";
+  Req.AnyScope = true;
+
+  EXPECT_THAT(match(*Idx, Req), IsEmpty());
+
+  Config C;
+  C.Index.External.emplace();
+  C.Index.External->Location = "test";
+  WithContextValue With(Config::Key, std::move(C));
+  EXPECT_THAT(match(*Idx, Req), ElementsAre("1"));
+  return;
+}
+
+TEST(ProjectAware, CreatedOnce) {
+  unsigned InvocationCount = 0;
+  IndexFactory Gen = [&](const Config::ExternalIndexSpec &, AsyncTaskRunner &) {
+++InvocationCount;
+return createIndex();
+  };
+
+  auto Idx = createProjectAwareIndex(std::move(Gen));
+  // No invocation at start.
+  EXPECT_EQ(InvocationCount, 0U);
+  FuzzyFindRequest Req;
+  Req.Query = "1";
+  Req.AnyScope = true;
+
+  // Cannot invoke without proper config.
+  match(*Idx, Req);
+  EXPECT_EQ(InvocationCount, 0U);
+
+  Config C;
+  C.Index.External.emplace();
+  C.Index.External->Location = "test";
+  WithContextValue With(Config::Key, std::move(C));
+  match(*Idx, Req);
+  // Now it should be created.
+  EXPECT_EQ(InvocationCount, 1U);
+  match(*Idx, Req);
+  // It is cached afterwards.
+  EXPECT_EQ(InvocationCount, 1U);
+  return;
+}
+} // namespace clangd
+} // namespace clang
Index: clang-tools-extra/clangd/unittests/CMakeLists.txt
===
--- clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -73,6 +73,7 @@
   PathMappingTests.cpp
   PreambleTests.cpp
   PrintASTTests.cpp
+  ProjectAwareIndexTests.cpp
   QualityTests.cpp
   RenameTests.cpp
   RIFFTests.cpp
Index: clang-tools-extra/clangd/index/ProjectAware.h
===
--- /dev/null
+++ clang-tools-extra/clangd/index/ProjectAware.h
@@ -0,0 +1,34 @@
+//===--- ProjectAware.h --*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_PROJECT_AWARE_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_PROJECT_AWARE_H
+
+#include "Config.h"
+#include "index/Index.h"
+#include "support/Threading.h"
+#include 
+#include 
+namespace clang {
+namespace clangd {
+
+/// A functor to create an index for an external index specification. Functor
+/// should perform any high latency operation in a separate thread through
+/// AsyncTaskRunner.
+using IndexFactory = std::function(
+const Config::ExternalIndexSpec &, AsyncTaskRunner &)>;
+
+/// Returns an index that answers queries using external indices. IndexGenerator
+/// can be used to customize how to generate an index from an external source.
+/// The default implementation loads the index asynchronously on the
+/// AsyncTaskRun

[PATCH] D90750: [clangd] Introduce ProjectAwareIndex

[Sam McCall via Phabricator via cfe-commits]

sammccall accepted this revision.
sammccall added inline comments.
This revision is now accepted and ready to land.



Comment at: clang-tools-extra/clangd/Config.h:95
+  static inline ExternalIndexSpec getEmptyKey() {
+static ExternalIndexSpec E{ExternalIndexSpec::File, "", ""};
+return E;

(nit: static isn't saving anything here I think: cost of constructing == cost 
of copying)


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D90750/new/

https://reviews.llvm.org/D90750

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D90750: [clangd] Introduce ProjectAwareIndex

[Kadir Cetinkaya via Phabricator via cfe-commits]

kadircet updated this revision to Diff 306615.
kadircet marked 6 inline comments as done.
kadircet added a comment.

- Internalize synchronization of indexstorage rather than using Memoize


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D90750/new/

https://reviews.llvm.org/D90750

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/Config.h
  clang-tools-extra/clangd/index/ProjectAware.cpp
  clang-tools-extra/clangd/index/ProjectAware.h
  clang-tools-extra/clangd/unittests/CMakeLists.txt
  clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp

Index: clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp
===
--- /dev/null
+++ clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp
@@ -0,0 +1,86 @@
+//===-- ProjectAwareIndexTests.cpp  ---*- C++ -*---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+
+#include "Config.h"
+#include "TestIndex.h"
+#include "index/Index.h"
+#include "index/MemIndex.h"
+#include "index/ProjectAware.h"
+#include "index/Ref.h"
+#include "index/Relation.h"
+#include "support/Context.h"
+#include "support/Threading.h"
+#include "llvm/ADT/StringRef.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include 
+#include 
+
+namespace clang {
+namespace clangd {
+using testing::ElementsAre;
+using testing::IsEmpty;
+
+std::unique_ptr createIndex() {
+  std::vector Symbols = {symbol("1")};
+  return std::make_unique(std::move(Symbols), RefSlab(),
+RelationSlab());
+}
+
+TEST(ProjectAware, Test) {
+  IndexFactory Gen = [](const Config::ExternalIndexSpec &, AsyncTaskRunner &) {
+return createIndex();
+  };
+
+  auto Idx = createProjectAwareIndex(std::move(Gen));
+  FuzzyFindRequest Req;
+  Req.Query = "1";
+  Req.AnyScope = true;
+
+  EXPECT_THAT(match(*Idx, Req), IsEmpty());
+
+  Config C;
+  C.Index.External.emplace();
+  C.Index.External->Location = "test";
+  WithContextValue With(Config::Key, std::move(C));
+  EXPECT_THAT(match(*Idx, Req), ElementsAre("1"));
+  return;
+}
+
+TEST(ProjectAware, CreatedOnce) {
+  unsigned InvocationCount = 0;
+  IndexFactory Gen = [&](const Config::ExternalIndexSpec &, AsyncTaskRunner &) {
+++InvocationCount;
+return createIndex();
+  };
+
+  auto Idx = createProjectAwareIndex(std::move(Gen));
+  // No invocation at start.
+  EXPECT_EQ(InvocationCount, 0U);
+  FuzzyFindRequest Req;
+  Req.Query = "1";
+  Req.AnyScope = true;
+
+  // Cannot invoke without proper config.
+  match(*Idx, Req);
+  EXPECT_EQ(InvocationCount, 0U);
+
+  Config C;
+  C.Index.External.emplace();
+  C.Index.External->Location = "test";
+  WithContextValue With(Config::Key, std::move(C));
+  match(*Idx, Req);
+  // Now it should be created.
+  EXPECT_EQ(InvocationCount, 1U);
+  match(*Idx, Req);
+  // It is cached afterwards.
+  EXPECT_EQ(InvocationCount, 1U);
+  return;
+}
+} // namespace clangd
+} // namespace clang
Index: clang-tools-extra/clangd/unittests/CMakeLists.txt
===
--- clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -73,6 +73,7 @@
   PathMappingTests.cpp
   PreambleTests.cpp
   PrintASTTests.cpp
+  ProjectAwareIndexTests.cpp
   QualityTests.cpp
   RenameTests.cpp
   RIFFTests.cpp
Index: clang-tools-extra/clangd/index/ProjectAware.h
===
--- /dev/null
+++ clang-tools-extra/clangd/index/ProjectAware.h
@@ -0,0 +1,34 @@
+//===--- ProjectAware.h --*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_PROJECT_AWARE_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_PROJECT_AWARE_H
+
+#include "Config.h"
+#include "index/Index.h"
+#include "support/Threading.h"
+#include 
+#include 
+namespace clang {
+namespace clangd {
+
+/// A functor to create an index for an external index specification. Functor
+/// should perform any high latency operation in a separate thread through
+/// AsyncTaskRunner.
+using IndexFactory = std::function(
+const Config::ExternalIndexSpec &, AsyncTaskRunner &)>;
+
+/// Returns an index that answers queries using external indices. IndexGenerator
+/// can be used to customize how to generate an index from an external source.
+/// The default implementation lo

[PATCH] D90750: [clangd] Introduce ProjectAwareIndex

[Sam McCall via Phabricator via cfe-commits]

sammccall added inline comments.



Comment at: clang-tools-extra/clangd/Config.h:88
 
+inline bool operator<(const Config::ExternalIndexSpec &LHS,
+  const Config::ExternalIndexSpec &RHS) {

Maybe we could use DenseMap instead? Defining an order on specs seems 
semantically iffy.

(Maybe one day we'll get to use a better hashtable and DenseMapInfo won't be so 
gross)



Comment at: clang-tools-extra/clangd/index/ProjectAware.cpp:137
+  if (!Gen) {
+Gen = [](const Config::ExternalIndexSpec &External,
+ AsyncTaskRunner &Tasks) {

nit: I think this would be clearer as a named function instead of a lambda



Comment at: clang-tools-extra/clangd/index/ProjectAware.cpp:100
+  });
+  addIndex(std::move(NewIndex));
+  return PlaceHolder;

kadircet wrote:
> sammccall wrote:
> > from memoize:
> > 
> > > Concurrent calls for the same key may run the
> > > computation multiple times, but each call will return the same result.
> > 
> > so this side-effect may run multiple times too, resulting in multiple index 
> > copies getting leaked. This seems like it may be bad in the case of file 
> > indexes.
> > 
> > Why can't the memoize map own the indexes?
> > This would still result in loading the index multiple times and throwing 
> > things away, hmm. Not sure how best to resolve this.
> > so this side-effect may run multiple times too, resulting in multiple index 
> > copies getting leaked. This seems like it may be bad in the case of file 
> > indexes.
> 
> The only way i can see this happening is by synchronizing calls to `getIndex`.
> 
> > Why can't the memoize map own the indexes?
> 
> Main problem is, it returns a copy to the stored elements, rather than a 
> reference to them. Which might be necessary in general to guarantee 
> thread-safety. Since we can't copy unique_ptrs, I had to store them elsewhere 
> and only keep pointers to it within the memoize map.
> 
> 
> So both things considered, maybe we should just give up on memoize and have 
> our own thread-safe cache here?
Yeah given that we know the compute function is going to be fast, we can just 
run it under the lock. So let's just use a big fat mutex.



Comment at: clang-tools-extra/clangd/index/ProjectAware.h:21
+/// A functor to create an index for an external index specification.
+using IndexGenerator = std::function(
+const Config::ExternalIndexSpec &, AsyncTaskRunner &)>;

nit: "factory" is more common than "generator" for this purpose, probably



Comment at: clang-tools-extra/clangd/index/ProjectAware.h:25
+/// Returns an index that answers queries using external indices. 
IndexGenerator
+/// can be used to customize how to generate an index from an external source.
+std::unique_ptr createProjectAwareIndex(IndexGenerator = nullptr);

Mention the default?

The default implementation loads the index asynchronously on the 
AsyncTaskRunner.
The index will appear empty until loaded.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D90750/new/

https://reviews.llvm.org/D90750

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D90750: [clangd] Introduce ProjectAwareIndex

[Kadir Cetinkaya via Phabricator via cfe-commits]

kadircet added a comment.

ping :)


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D90750/new/

https://reviews.llvm.org/D90750

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D90750: [clangd] Introduce ProjectAwareIndex

[Kadir Cetinkaya via Phabricator via cfe-commits]

kadircet added inline comments.



Comment at: clang-tools-extra/clangd/index/ProjectAware.cpp:100
+  });
+  addIndex(std::move(NewIndex));
+  return PlaceHolder;

sammccall wrote:
> from memoize:
> 
> > Concurrent calls for the same key may run the
> > computation multiple times, but each call will return the same result.
> 
> so this side-effect may run multiple times too, resulting in multiple index 
> copies getting leaked. This seems like it may be bad in the case of file 
> indexes.
> 
> Why can't the memoize map own the indexes?
> This would still result in loading the index multiple times and throwing 
> things away, hmm. Not sure how best to resolve this.
> so this side-effect may run multiple times too, resulting in multiple index 
> copies getting leaked. This seems like it may be bad in the case of file 
> indexes.

The only way i can see this happening is by synchronizing calls to `getIndex`.

> Why can't the memoize map own the indexes?

Main problem is, it returns a copy to the stored elements, rather than a 
reference to them. Which might be necessary in general to guarantee 
thread-safety. Since we can't copy unique_ptrs, I had to store them elsewhere 
and only keep pointers to it within the memoize map.


So both things considered, maybe we should just give up on memoize and have our 
own thread-safe cache here?


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D90750/new/

https://reviews.llvm.org/D90750

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D90750: [clangd] Introduce ProjectAwareIndex

[Kadir Cetinkaya via Phabricator via cfe-commits]

kadircet updated this revision to Diff 305695.
kadircet marked 5 inline comments as done.
kadircet added a comment.

- Address comments
- Only query the associated index
- Use ExternalIndexSpec as the cache key


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D90750/new/

https://reviews.llvm.org/D90750

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/Config.h
  clang-tools-extra/clangd/index/ProjectAware.cpp
  clang-tools-extra/clangd/index/ProjectAware.h
  clang-tools-extra/clangd/unittests/CMakeLists.txt
  clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp

Index: clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp
===
--- /dev/null
+++ clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp
@@ -0,0 +1,84 @@
+//===-- ProjectAwareIndexTests.cpp  ---*- C++ -*---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+
+#include "Config.h"
+#include "TestIndex.h"
+#include "index/Index.h"
+#include "index/MemIndex.h"
+#include "index/ProjectAware.h"
+#include "index/Ref.h"
+#include "index/Relation.h"
+#include "support/Context.h"
+#include "support/Threading.h"
+#include "llvm/ADT/StringRef.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include 
+#include 
+
+namespace clang {
+namespace clangd {
+using testing::ElementsAre;
+using testing::IsEmpty;
+
+std::unique_ptr createIndex() {
+  std::vector Symbols = {symbol("1")};
+  return std::make_unique(std::move(Symbols), RefSlab(),
+RelationSlab());
+}
+
+TEST(ProjectAware, Test) {
+  IndexGenerator Gen = [](const Config::ExternalIndexSpec &,
+  AsyncTaskRunner &) { return createIndex(); };
+
+  auto Idx = createProjectAwareIndex(std::move(Gen));
+  FuzzyFindRequest Req;
+  Req.Query = "1";
+  Req.AnyScope = true;
+
+  EXPECT_THAT(match(*Idx, Req), IsEmpty());
+
+  Config C;
+  C.Index.External.emplace();
+  WithContextValue With(Config::Key, std::move(C));
+  EXPECT_THAT(match(*Idx, Req), ElementsAre("1"));
+  return;
+}
+
+TEST(ProjectAware, CreatedOnce) {
+  unsigned InvocationCount = 0;
+  IndexGenerator Gen = [&InvocationCount](const Config::ExternalIndexSpec &,
+  AsyncTaskRunner &) {
+++InvocationCount;
+return createIndex();
+  };
+
+  auto Idx = createProjectAwareIndex(std::move(Gen));
+  // No invocation at start.
+  EXPECT_EQ(InvocationCount, 0U);
+  FuzzyFindRequest Req;
+  Req.Query = "1";
+  Req.AnyScope = true;
+
+  // Cannot invoke without proper config.
+  match(*Idx, Req);
+  EXPECT_EQ(InvocationCount, 0U);
+
+  Config C;
+  C.Index.External.emplace();
+  WithContextValue With(Config::Key, std::move(C));
+  match(*Idx, Req);
+  // Now it should be created.
+  EXPECT_EQ(InvocationCount, 1U);
+  match(*Idx, Req);
+  // It is cached afterwards.
+  EXPECT_EQ(InvocationCount, 1U);
+  return;
+}
+} // namespace clangd
+} // namespace clang
Index: clang-tools-extra/clangd/unittests/CMakeLists.txt
===
--- clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -73,6 +73,7 @@
   PathMappingTests.cpp
   PreambleTests.cpp
   PrintASTTests.cpp
+  ProjectAwareIndexTests.cpp
   QualityTests.cpp
   RenameTests.cpp
   RIFFTests.cpp
Index: clang-tools-extra/clangd/index/ProjectAware.h
===
--- /dev/null
+++ clang-tools-extra/clangd/index/ProjectAware.h
@@ -0,0 +1,30 @@
+//===--- ProjectAware.h --*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_PROJECT_AWARE_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_PROJECT_AWARE_H
+
+#include "Config.h"
+#include "index/Index.h"
+#include "support/Threading.h"
+#include 
+#include 
+namespace clang {
+namespace clangd {
+
+/// A functor to create an index for an external index specification.
+using IndexGenerator = std::function(
+const Config::ExternalIndexSpec &, AsyncTaskRunner &)>;
+
+/// Returns an index that answers queries using external indices. IndexGenerator
+/// can be used to customize how to generate an index from an external source.
+std::unique_ptr createProjectAwareIndex(IndexGenerator = nullptr);
+} // namespace clangd
+} // namespace clang
+

[PATCH] D90750: [clangd] Introduce ProjectAwareIndex

[Sam McCall via Phabricator via cfe-commits]

sammccall added a comment.

This looks really sensible to me! Nice you managed to reuse Memoize, there's 
very little ugly caching/locking left.




Comment at: clang-tools-extra/clangd/index/ProjectAware.cpp:81
+return IndexForProject.get(
+External.Location, [Server = External.Location,
+MP = llvm::StringRef(External.MountPoint), this] {

I think the cache key needs a bit more consideration.

It's unlikely that a filename will collide with a host/port, but they are 
currently sitting in the same namespace.

More worryingly, the mountpoint is used in the lambda body but not in the cache 
key - are we sure this is safe?

The clearest way to enforce this is to make index creation use the cache key 
only, and not other data from the config. Then you have to include what you're 
going to use.



Comment at: clang-tools-extra/clangd/index/ProjectAware.cpp:81
+return IndexForProject.get(
+External.Location, [Server = External.Location,
+MP = llvm::StringRef(External.MountPoint), this] {

sammccall wrote:
> I think the cache key needs a bit more consideration.
> 
> It's unlikely that a filename will collide with a host/port, but they are 
> currently sitting in the same namespace.
> 
> More worryingly, the mountpoint is used in the lambda body but not in the 
> cache key - are we sure this is safe?
> 
> The clearest way to enforce this is to make index creation use the cache key 
> only, and not other data from the config. Then you have to include what 
> you're going to use.
as discussed offline, allowing the config -> cache key and cache key -> index 
operations to be injectable is the key to testing I think



Comment at: clang-tools-extra/clangd/index/ProjectAware.cpp:100
+  });
+  addIndex(std::move(NewIndex));
+  return PlaceHolder;

from memoize:

> Concurrent calls for the same key may run the
> computation multiple times, but each call will return the same result.

so this side-effect may run multiple times too, resulting in multiple index 
copies getting leaked. This seems like it may be bad in the case of file 
indexes.

Why can't the memoize map own the indexes?
This would still result in loading the index multiple times and throwing things 
away, hmm. Not sure how best to resolve this.



Comment at: clang-tools-extra/clangd/index/ProjectAware.cpp:119
+  SymbolIndex *Result = nullptr;
+  auto PushIndex = [this, &Result](SymbolIndex *Idx) {
+if (!Result) {

consider moving this to `unique_ptr 
MergedIndex::create(ArrayRef>)` or so?

Not sure if this is something we'd want to use from ClangdMain...



Comment at: clang-tools-extra/clangd/index/ProjectAware.cpp:128
+  std::lock_guard Lock(Mu);
+  for (const auto &Idx : IndexStorage) {
+if (Idx.get() == Primary)

with the helper described above, the Primary ordering could be maybe better 
expressed as std::stable_partition



Comment at: clang-tools-extra/clangd/index/ProjectAware.h:30
+
+class ProjectAwareIndex : public SymbolIndex {
+public:

Is there a reason this class needs to be public?
I think we can expose factories returning SymbolIndex only


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D90750/new/

https://reviews.llvm.org/D90750

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D90750: [clangd] Introduce ProjectAwareIndex

[Sam McCall via Phabricator via cfe-commits]

sammccall added a comment.

In D90750#2378987 , @kadircet wrote:

> The index has the following query semantics:

Rehashing offline conversation:

- These heuristics mostly amount to guessing which high-level operation is 
being performed. It's a non-obvious place to put this logic, and it's hard to 
fix without fallout when the guess is wrong.
- These guesses will get stale. e.g. this queries all indexes for relations, 
but one index for lookup. We'd like to use both for Hover soon.

Our conclusion was we could query one index by default, and allow callers to 
override this using a context variable.
The arguments for a context variable vs attributes on request:

- it naturally applies to batches of requests
- it's a less invasive change
- it "hides" the option from other indexes

I'm not that sure about this implementation choice, but it also probably 
doesn't matter much.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D90750/new/

https://reviews.llvm.org/D90750

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D90750: [clangd] Introduce ProjectAwareIndex

[Kadir Cetinkaya via Phabricator via cfe-commits]

kadircet updated this revision to Diff 304795.
kadircet added a comment.

- Rebase


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D90750/new/

https://reviews.llvm.org/D90750

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/index/ProjectAware.cpp
  clang-tools-extra/clangd/index/ProjectAware.h
  clang-tools-extra/clangd/unittests/CMakeLists.txt
  clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp

Index: clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp
===
--- /dev/null
+++ clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp
@@ -0,0 +1,10 @@
+//===-- ProjectAwareIndexTests.cpp  ---*- C++ -*---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
Index: clang-tools-extra/clangd/unittests/CMakeLists.txt
===
--- clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -73,6 +73,7 @@
   PathMappingTests.cpp
   PreambleTests.cpp
   PrintASTTests.cpp
+  ProjectAwareIndexTests.cpp
   QualityTests.cpp
   RenameTests.cpp
   RIFFTests.cpp
Index: clang-tools-extra/clangd/index/ProjectAware.h
===
--- /dev/null
+++ clang-tools-extra/clangd/index/ProjectAware.h
@@ -0,0 +1,74 @@
+//===--- ProjectAware.h --*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_PROJECT_AWARE_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_PROJECT_AWARE_H
+
+#include "Config.h"
+#include "index/Index.h"
+#include "index/MemIndex.h"
+#include "index/Merge.h"
+#include "index/Serialization.h"
+#include "index/remote/Client.h"
+#include "support/Threading.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+namespace clang {
+namespace clangd {
+
+class ProjectAwareIndex : public SymbolIndex {
+public:
+  size_t estimateMemoryUsage() const override;
+
+  /// Only queries the associated index with the current context.
+  void lookup(const LookupRequest &Req,
+  llvm::function_ref Callback) const override;
+
+  /// Query all indexes while prioritizing the associated one (if any).
+  bool refs(const RefsRequest &Req,
+llvm::function_ref Callback) const override;
+
+  /// Queries only the associates index when Req.RestrictForCodeCompletion is
+  /// set, otherwise queries all.
+  bool
+  fuzzyFind(const FuzzyFindRequest &Req,
+llvm::function_ref Callback) const override;
+
+  /// Query all indexes while prioritizing the associated one (if any).
+  void relations(const RelationsRequest &Req,
+ llvm::function_ref
+ Callback) const override;
+
+private:
+  // Returns the index associated with current context, if any.
+  SymbolIndex *getIndex() const;
+  // Adds the Idx into IndexStorage and adjusts the MergedIdx view. Returns a
+  // pointer to the Idx.
+  SymbolIndex *addIndex(std::unique_ptr Idx) const;
+  // Creates a view over all indexes, putting Primary at the top.
+  SymbolIndex *mergedView(SymbolIndex *Primary) const;
+
+  // Storage for all the external indexes.
+  Memoize> IndexForProject;
+  mutable std::mutex Mu;
+  mutable std::vector> IndexStorage;
+  mutable AsyncTaskRunner Tasks;
+  // Used for creating a merged view over all the indexes seen so far to ease
+  // querying them all at once.
+  mutable std::vector> MergedIdxStorage;
+};
+} // namespace clangd
+} // namespace clang
+
+#endif
Index: clang-tools-extra/clangd/index/ProjectAware.cpp
===
--- /dev/null
+++ clang-tools-extra/clangd/index/ProjectAware.cpp
@@ -0,0 +1,138 @@
+//===--- ProjectAware.h --*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+
+#include "ProjectAware.h"
+#include "index/Index.h"
+#include "index/Merge.h"
+#include "index/Ref.h"
+#i

[PATCH] D90750: [clangd] Introduce ProjectAwareIndex

[Kadir Cetinkaya via Phabricator via cfe-commits]

kadircet updated this revision to Diff 303442.
kadircet added a comment.

The index has the following query semantics:

- Lookup: it only queries the associated index. This is to not regress

latency for operations like Hover and Go-To.

- FuzzyFind only queries the associated index when

RestrictForCodeCompletion is set, it queries all otherwise. Again this
is to prevent latency regression for code completion, but make sure we
provide complete results for search operations like WorkspaceSymbol.

- Refs, queries all. As incomplete results for such operations might

result in bad UX for opeartions like rename. Also having an incomplete
set of references might result in bad impressions.

- Relations, queries all. Again this is used by operations like type and

call hierarchy and having incomplete results are useless for such
operations.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D90750/new/

https://reviews.llvm.org/D90750

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/index/ProjectAware.cpp
  clang-tools-extra/clangd/index/ProjectAware.h
  clang-tools-extra/clangd/unittests/CMakeLists.txt
  clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp

Index: clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp
===
--- /dev/null
+++ clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp
@@ -0,0 +1,7 @@
+//===-- ProjectAwareIndexTests.cpp  ---*- C++ -*---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
Index: clang-tools-extra/clangd/unittests/CMakeLists.txt
===
--- clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -73,6 +73,7 @@
   PathMappingTests.cpp
   PreambleTests.cpp
   PrintASTTests.cpp
+  ProjectAwareIndexTests.cpp
   QualityTests.cpp
   RenameTests.cpp
   RIFFTests.cpp
Index: clang-tools-extra/clangd/index/ProjectAware.h
===
--- /dev/null
+++ clang-tools-extra/clangd/index/ProjectAware.h
@@ -0,0 +1,76 @@
+//===--- ProjectAware.h --*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_PROJECT_AWARE_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_PROJECT_AWARE_H
+
+#include "Config.h"
+#include "index/Index.h"
+#include "index/MemIndex.h"
+#include "index/Merge.h"
+#include "index/Serialization.h"
+#include "index/remote/Client.h"
+#include "support/Threading.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+namespace clang {
+namespace clangd {
+
+class ProjectAwareIndex : public SymbolIndex {
+public:
+  size_t estimateMemoryUsage() const override;
+
+  /// Only queries the associated index with the current context.
+  void lookup(const LookupRequest &Req,
+  llvm::function_ref Callback) const override;
+
+  /// Query all indexes while prioritizing the associated one (if any).
+  bool refs(const RefsRequest &Req,
+llvm::function_ref Callback) const override;
+
+  /// Queries only the associates index when Req.RestrictForCodeCompletion is
+  /// set, otherwise queries all.
+  bool
+  fuzzyFind(const FuzzyFindRequest &Req,
+llvm::function_ref Callback) const override;
+
+  /// Query all indexes while prioritizing the associated one (if any).
+  void relations(const RelationsRequest &Req,
+ llvm::function_ref
+ Callback) const override;
+
+private:
+  // Returns the index associated with current context, if any.
+  SymbolIndex *getIndex() const;
+  // Adds the Idx into IndexStorage and adjusts the MergedIdx view. Returns a
+  // pointer to the Idx.
+  SymbolIndex *addIndex(std::unique_ptr Idx) const;
+
+  // Storage for all the external indexes.
+  Memoize> IndexForProject;
+  mutable std::mutex Mu;
+  mutable std::vector> IndexStorage;
+  AsyncTaskRunner Tasks;
+
+  // Used for creating a merged view over all the indexes seen so far to ease
+  // querying them all at once.
+  mutable std::mutex MergedIdxMu;
+  mutable std::vector> MergedIdxStorage;
+  // Points at root of merged view of all smybols.
+  mutable SymbolIndex *MergedIdx = nullptr;
+};
+} // namespace clangd
+} // names

[PATCH] D90750: [clangd] Introduce ProjectAwareIndex

[Kadir Cetinkaya via Phabricator via cfe-commits]

kadircet planned changes to this revision.
kadircet added a comment.

This is a WIP patch. Provided as a way to demonstrate how `ExternalBlock` will 
be used in an index implementation.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D90750/new/

https://reviews.llvm.org/D90750

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D90750: [clangd] Introduce ProjectAwareIndex

[Kadir Cetinkaya via Phabricator via cfe-commits]

kadircet created this revision.
kadircet added a reviewer: sammccall.
Herald added subscribers: cfe-commits, usaxena95, arphaman, mgorny.
Herald added a project: clang.
kadircet requested review of this revision.
Herald added subscribers: MaskRay, ilya-biryukov.

An index implementation that can dispatch to a variety of indexes
depending on the file path. Enables clangd to work with multiple indexes in the
same instance, configured via config files.

Depends on D90749 , D90746 



Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D90750

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/index/ProjectAware.cpp
  clang-tools-extra/clangd/index/ProjectAware.h
  clang-tools-extra/clangd/unittests/CMakeLists.txt
  clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp

Index: clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp
===
--- /dev/null
+++ clang-tools-extra/clangd/unittests/ProjectAwareIndexTests.cpp
@@ -0,0 +1,7 @@
+//===-- ProjectAwareIndexTests.cpp  ---*- C++ -*---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
Index: clang-tools-extra/clangd/unittests/CMakeLists.txt
===
--- clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -73,6 +73,7 @@
   PathMappingTests.cpp
   PreambleTests.cpp
   PrintASTTests.cpp
+  ProjectAwareIndexTests.cpp
   QualityTests.cpp
   RenameTests.cpp
   RIFFTests.cpp
Index: clang-tools-extra/clangd/index/ProjectAware.h
===
--- /dev/null
+++ clang-tools-extra/clangd/index/ProjectAware.h
@@ -0,0 +1,59 @@
+//===--- ProjectAware.h --*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_PROJECT_AWARE_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_PROJECT_AWARE_H
+
+#include "Config.h"
+#include "index/Index.h"
+#include "index/MemIndex.h"
+#include "index/Merge.h"
+#include "index/Serialization.h"
+#include "index/remote/Client.h"
+#include "support/Threading.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include 
+#include 
+#include 
+#include 
+#include 
+namespace clang {
+namespace clangd {
+
+class ProjectAwareIndex : public SymbolIndex {
+public:
+  size_t estimateMemoryUsage() const override;
+
+  void lookup(const LookupRequest &Req,
+  llvm::function_ref Callback) const override;
+
+  bool refs(const RefsRequest &Req,
+llvm::function_ref Callback) const override;
+
+  bool
+  fuzzyFind(const FuzzyFindRequest &Req,
+llvm::function_ref Callback) const override;
+
+  void relations(const RelationsRequest &Req,
+ llvm::function_ref
+ Callback) const override;
+
+private:
+  SymbolIndex *getIndex() const;
+
+  Memoize> IndexForProject;
+  mutable std::mutex Mu;
+  mutable std::list> IndexStorage;
+  AsyncTaskRunner Tasks;
+};
+} // namespace clangd
+} // namespace clang
+
+#endif
Index: clang-tools-extra/clangd/index/ProjectAware.cpp
===
--- /dev/null
+++ clang-tools-extra/clangd/index/ProjectAware.cpp
@@ -0,0 +1,88 @@
+//===--- ProjectAware.h --*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+
+#include "ProjectAware.h"
+#include "support/Logger.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace clang {
+namespace clangd {
+size_t ProjectAwareIndex::estimateMemoryUsage() const {
+  size_t Total = 0;
+  std::lock_guard Lock(Mu);
+  for (auto &Idx : IndexStorage)
+Total += Idx->estimateMemoryUsage();
+  return Total;
+}
+
+void ProjectAwareIndex::lookup(
+const LookupRequest &Req,
+llvm::function_ref Callback) const {
+  if (auto *Idx = getIndex())
+Idx->lookup(Req, Callback);
+}
+
+bool ProjectAwareIndex::refs(
+const RefsRequest &Req,
+llvm::function_ref Callback) const {
+  if (auto *Idx = getIndex())