This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch smoosh-cleanup-search
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit d2e9a63b8048a350fea3bccd1521e9d72fdb1665
Author: Robert Newson <rnew...@apache.org>
AuthorDate: Thu Aug 3 18:32:56 2023 +0100
WIP enhance smoosh to cleanup search indexes when ddocs change
---
src/fabric/src/fabric.erl | 21 ++++++++++++++-------
src/nouveau/src/nouveau_fabric_cleanup.erl | 9 +--------
src/nouveau/src/nouveau_util.erl | 15 +++++++++++++++
src/smoosh/src/smoosh_server.erl | 5 +++++
4 files changed, 35 insertions(+), 15 deletions(-)
diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl
index dfd433d38..e076506e8 100644
--- a/src/fabric/src/fabric.erl
+++ b/src/fabric/src/fabric.erl
@@ -590,25 +590,32 @@ cleanup_index_files(DbName) ->
cleanup_local_indices_and_purge_checkpoints([]) ->
ok;
cleanup_local_indices_and_purge_checkpoints([_ | _] = Dbs) ->
- AllIndices = lists:map(fun couch_mrview_util:get_index_files/1, Dbs),
- AllPurges = lists:map(fun couch_mrview_util:get_purge_checkpoints/1, Dbs),
- Sigs = couch_mrview_util:get_signatures(hd(Dbs)),
- ok = cleanup_purges(Sigs, AllPurges, Dbs),
- ok = cleanup_indices(Sigs, AllIndices).
+ MrViewIndices = lists:map(fun couch_mrview_util:get_index_files/1, Dbs),
+ MrViewPurges = lists:map(fun couch_mrview_util:get_purge_checkpoints/1,
Dbs),
+ MrViewSigs = couch_mrview_util:get_signatures(hd(Dbs)),
+ ok = cleanup_mrview_purges(MrViewSigs, MrViewPurges, Dbs),
+ ok = cleanup_mrview_indices(MrViewSigs, MrViewIndices),
-cleanup_purges(Sigs, AllPurges, Dbs) ->
+ NouveauSigs = nouveau_util:active_sigs(hd(Dbs)),
+ ok = cleanup_nouveau_indices(Dbs, NouveauSigs).
+
+cleanup_mrview_purges(Sigs, AllPurges, Dbs) ->
Fun = fun(DbPurges, Db) ->
couch_mrview_cleanup:cleanup_purges(Db, Sigs, DbPurges)
end,
lists:zipwith(Fun, AllPurges, Dbs),
ok.
-cleanup_indices(Sigs, AllIndices) ->
+cleanup_mrview_indices(Sigs, AllIndices) ->
Fun = fun(DbIndices) ->
couch_mrview_cleanup:cleanup_indices(Sigs, DbIndices)
end,
lists:foreach(Fun, AllIndices).
+cleanup_nouveau_indices(Dbs, ActiveSigs) ->
+ Fun = fun(Db) -> nouveau_api:delete_path(nouveau_util:index_name(Db),
ActiveSigs) end,
+ lists:foreach(Fun, Dbs).
+
%% @doc clean up index files for a specific db on all nodes
-spec cleanup_index_files_all_nodes(dbname()) -> [reference()].
cleanup_index_files_all_nodes(DbName) ->
diff --git a/src/nouveau/src/nouveau_fabric_cleanup.erl
b/src/nouveau/src/nouveau_fabric_cleanup.erl
index cd4128fb1..e1567ef8d 100644
--- a/src/nouveau/src/nouveau_fabric_cleanup.erl
+++ b/src/nouveau/src/nouveau_fabric_cleanup.erl
@@ -14,9 +14,6 @@
-module(nouveau_fabric_cleanup).
--include_lib("couch/include/couch_db.hrl").
-
--include("nouveau.hrl").
-include_lib("mem3/include/mem3.hrl").
-export([go/1]).
@@ -26,7 +23,7 @@ go(DbName) ->
ActiveSigs =
lists:usort(
lists:flatmap(
- fun(Doc) -> active_sigs(DbName, Doc) end,
+ fun(Doc) -> nouveau_util:active_sigs(DbName, Doc) end,
[couch_doc:from_json_obj(DD) || DD <- DesignDocs]
)
),
@@ -37,7 +34,3 @@ go(DbName) ->
end,
Shards
).
-
-active_sigs(DbName, #doc{} = Doc) ->
- Indexes = nouveau_util:design_doc_to_indexes(DbName, Doc),
- lists:map(fun(Index) -> Index#index.sig end, Indexes).
diff --git a/src/nouveau/src/nouveau_util.erl b/src/nouveau/src/nouveau_util.erl
index b7da7e802..88fda6d30 100644
--- a/src/nouveau/src/nouveau_util.erl
+++ b/src/nouveau/src/nouveau_util.erl
@@ -22,6 +22,8 @@
index_name/1,
design_doc_to_indexes/2,
design_doc_to_index/3,
+ active_sigs/1,
+ active_sigs/2,
nouveau_url/0
]).
@@ -93,5 +95,18 @@ design_doc_to_index(DbName, #doc{id = Id, body = {Fields}},
IndexName) ->
{error, InvalidDDocError}
end.
+active_sigs(DbName) when is_binary(DbName) ->
+ {ok, DesignDocs} = fabric:design_docs(mem3:dbname(DbName)),
+ lists:usort(
+ lists:flatmap(
+ fun(Doc) -> active_sigs(DbName, Doc) end,
+ [couch_doc:from_json_obj(DD) || DD <- DesignDocs]
+ )
+ ).
+
+active_sigs(DbName, #doc{} = Doc) ->
+ Indexes = design_doc_to_indexes(DbName, Doc),
+ lists:map(fun(Index) -> Index#index.sig end, Indexes).
+
nouveau_url() ->
config:get("nouveau", "url", "http://127.0.0.1:8080";).
diff --git a/src/smoosh/src/smoosh_server.erl b/src/smoosh/src/smoosh_server.erl
index 10368a549..5e7314dff 100644
--- a/src/smoosh/src/smoosh_server.erl
+++ b/src/smoosh/src/smoosh_server.erl
@@ -123,6 +123,9 @@ handle_db_event(DbName, local_updated, St) ->
handle_db_event(DbName, updated, St) ->
enqueue(DbName),
{ok, St};
+handle_db_event(DbName, ddoc_updated, St) ->
+ enqueue({?INDEX_CLEANUP, mem3:dbname(DbName)}),
+ {ok, St};
handle_db_event(DbName, {index_commit, IdxName}, St) ->
enqueue({DbName, IdxName}),
{ok, St};
@@ -342,6 +345,8 @@ find_channel(#state{} = State, [Channel | Rest], Object) ->
find_channel(State, Rest, Object)
end.
+stale_enough({?INDEX_CLEANUP, _}) ->
+ true;
stale_enough(Object) ->
LastUpdatedSec = last_updated(Object),
Staleness = erlang:monotonic_time(second) - LastUpdatedSec,
