This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new f83dd8d74 Fix stuck mem3 purge checkpoints
f83dd8d74 is described below
commit f83dd8d74cf197fd67ff4de7447d5620088c27c1
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Sat Nov 7 18:40:04 2026 -0500
Fix stuck mem3 purge checkpoints
Observed a few rare cases of purges being stuck in production
We had the wrong field name in read repair purge checkpoint filter. Before
we
looked up `target_node` in the purge checkpoint docs but it should really be
`target` (see mem3_rep:purge_cp_body/3). This way every node started at
purge
seq 0.
`pull_purges/5` had the target/source swapped. The checkpoint lives on the
target
so we should always write it from the target's perspective, meaning the
target
is the source and we're the target.
There was an off-by-one error in `have_all_purge_checkpoints/1`
Shards db checkpoints have an undefined range (since they are effectively
local
dbs replicated in special ring topology) and they replicated by name. As
such
they never matched the `{Node, [0, ?RING_END]}` copies so
have_all_purge_checkpoints/1 was always false for them.
Fortunately this is easy to fix, and just deleting the checkpoints and
letting
the next internal replicator cycle start over from the oldest purge seq
works.
---
src/couch/src/couch_bt_engine_compactor.erl | 5 +-
.../test/eunit/couch_bt_engine_compactor_tests.erl | 33 ++++++++++-
src/fabric/src/fabric_rpc.erl | 12 +++-
src/fabric/test/eunit/fabric_rpc_purge_tests.erl | 51 ++++++++++++++++-
src/mem3/src/mem3_rep.erl | 27 +++++++--
src/mem3/test/eunit/mem3_rep_test.erl | 65 +++++++++++++++++++++-
6 files changed, 179 insertions(+), 14 deletions(-)
diff --git a/src/couch/src/couch_bt_engine_compactor.erl
b/src/couch/src/couch_bt_engine_compactor.erl
index 36c149217..2a8de05cf 100644
--- a/src/couch/src/couch_bt_engine_compactor.erl
+++ b/src/couch/src/couch_bt_engine_compactor.erl
@@ -176,9 +176,12 @@ copy_purge_info(#comp_st{} = CompSt) ->
% that case we'd end up with a "hole" between 500 and 1001 -- a new
% minimum purge sequence of 500, but the oldest checkpoint is would be
% 1001.
+ %
+ % Keep infos with PSeq > MinPurgeSeq. To get all of them, including the
+ % oldest need to use Oldest-1.
case mem3_rep:have_all_purge_checkpoints(Db) of
true -> couch_db:get_minimum_purge_seq(Db);
- false -> couch_db:get_oldest_purge_seq(Db)
+ false -> max(0, couch_db:get_oldest_purge_seq(Db) - 1)
end
end),
OldPSTree = OldSt#st.purge_seq_tree,
diff --git a/src/couch/test/eunit/couch_bt_engine_compactor_tests.erl
b/src/couch/test/eunit/couch_bt_engine_compactor_tests.erl
index 63df30fb0..69667a7e4 100644
--- a/src/couch/test/eunit/couch_bt_engine_compactor_tests.erl
+++ b/src/couch/test/eunit/couch_bt_engine_compactor_tests.erl
@@ -41,7 +41,8 @@ basic_compaction_test_() ->
fun teardown/1,
[
fun compaction_resume/1,
- fun is_compacting_works/1
+ fun is_compacting_works/1,
+ fun preserve_purges_without_all_checkpoints/1
]
}
}.
@@ -81,6 +82,36 @@ is_compacting_works(DbName) ->
meck:unload(couch_emsort)
end).
+preserve_purges_without_all_checkpoints(DbName) ->
+ ?_test(begin
+ purge_docs(DbName),
+ ?assertEqual({1, 3, 3}, purge_info_stats(DbName)),
+ meck:new(mem3_rep, [passthrough]),
+ meck:expect(mem3_rep, have_all_purge_checkpoints, 1, false),
+ compact_db(DbName),
+ ?assert(meck:called(mem3_rep, have_all_purge_checkpoints, 1)),
+ ?assertEqual({1, 3, 3}, purge_info_stats(DbName)),
+ meck:unload(mem3_rep)
+ end).
+
+purge_docs(DbName) ->
+ couch_util:with_db(DbName, fun(Db) ->
+ PurgeFun = fun(DocId) ->
+ {ok, #doc{revs = {Pos, [Rev | _]}}} = couch_db:open_doc(Db, DocId,
[]),
+ {couch_uuids:random(), DocId, [{Pos, Rev}]}
+ end,
+ PInfos = lists:map(PurgeFun, [~"doc1", ~"doc2", ~"doc3"]),
+ {ok, _} = couch_db:purge_docs(Db, PInfos)
+ end).
+
+% Return {OldestPurgeSeq, PurgeSeq, PurgeInfosCount}
+purge_info_stats(DbName) ->
+ couch_util:with_db(DbName, fun(Db) ->
+ FoldFun = fun(_, Acc) -> {ok, Acc + 1} end,
+ {ok, Count} = couch_db:fold_purge_infos(Db, FoldFun, 0),
+ {couch_db:get_oldest_purge_seq(Db), couch_db:get_purge_seq(Db), Count}
+ end).
+
check_db_validity(DbName) ->
couch_util:with_db(DbName, fun(Db) ->
?assertEqual({ok, 3}, couch_db:get_doc_count(Db)),
diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl
index a05d59f96..1a2bc4928 100644
--- a/src/fabric/src/fabric_rpc.erl
+++ b/src/fabric/src/fabric_rpc.erl
@@ -582,14 +582,18 @@ read_repair_filter(Db, Docs, NodeRevs) ->
NodeSeqs = get_node_seqs(Db, Nodes),
DbPSeq = couch_db:get_purge_seq(Db),
+ OldestPSeq = couch_db:get_oldest_purge_seq(Db),
Lag = config:get_integer("couchdb", "read_repair_lag", 100),
% Filter out read-repair updates from any node that is
% so out of date that it would force us to scan a large
- % number of purge infos
+ % number of purge infos. And drop nodes behind our oldest purge info.
+ % Compaction might already removed infos they haven't seen, so we can't
tell if
+ % their revisions were purged, and fold_purge_infos/4 would
+ % fail with an invalid_start_purge_seq error.
NodeFiltFun = fun({Node, _Revs}) ->
{Node, NodeSeq} = lists:keyfind(Node, 1, NodeSeqs),
- NodeSeq >= DbPSeq - Lag
+ NodeSeq >= DbPSeq - Lag andalso NodeSeq >= OldestPSeq - 1
end,
RecentNodeRevs = lists:filter(NodeFiltFun, NonLocalNodeRevs),
@@ -625,7 +629,9 @@ get_node_seqs(Db, Nodes) ->
FoldFun = fun(#doc{id = Id, body = {Props}}, Acc) ->
case Id of
<<?LOCAL_DOC_PREFIX, "purge-mem3-", _/binary>> ->
- TgtNode = couch_util:get_value(<<"target_node">>, Props),
+ % Should match mem3_rep:purge_cp_body/3. There "target"
+ % is the shard we replicate our purges TO.
+ TgtNode = couch_util:get_value(~"target", Props),
PurgeSeq = couch_util:get_value(<<"purge_seq">>, Props),
case lists:keyfind(TgtNode, 1, Acc) of
{_, OldSeq} ->
diff --git a/src/fabric/test/eunit/fabric_rpc_purge_tests.erl
b/src/fabric/test/eunit/fabric_rpc_purge_tests.erl
index fa6123dfc..3d2eb35b2 100644
--- a/src/fabric/test/eunit/fabric_rpc_purge_tests.erl
+++ b/src/fabric/test/eunit/fabric_rpc_purge_tests.erl
@@ -60,6 +60,19 @@ main_test_() ->
?TDEF_FE(t_no_filter_different_node),
?TDEF_FE(t_no_filter_after_repl)
]
+ },
+ {
+ foreach,
+ fun setup_trimmed_purge/0,
+ fun teardown_trimmed_purge/1,
+ [
+ ?TDEF_FE(t_filter),
+ ?TDEF_FE(t_filter_unknown_node),
+ ?TDEF_FE(t_filter_local_node),
+ ?TDEF_FE(t_no_filter_old_node),
+ ?TDEF_FE(t_no_filter_different_node),
+ ?TDEF_FE(t_no_filter_after_repl)
+ ]
}
]
}.
@@ -104,6 +117,21 @@ setup_multi_purge() ->
teardown_multi_purge(Ctx) ->
teardown_single_purge(Ctx).
+% Like the multi case above, but with the compactor trimmed oldest purge infos
+% 1 to 3. After that the oldest purge seq is 4. Nodes with checkpoints before 3
+% are too far behind and their read repair updates should be filtered out.
+setup_trimmed_purge() ->
+ {DbName, _, _, _} = Ctx = setup_multi_purge(),
+ couch_util:with_db(DbName, fun(Db) ->
+ ok = couch_db:set_purge_infos_limit(Db, 2)
+ end),
+ compact(DbName),
+ ?assertEqual({4, 5}, purge_seqs(DbName)),
+ Ctx.
+
+teardown_trimmed_purge(Ctx) ->
+ teardown_single_purge(Ctx).
+
t_no_purge_no_filter(DbName) ->
DocId = <<"0003">>,
@@ -239,18 +267,37 @@ create_purge_checkpoint(DbName, PurgeSeq, TgtNode) when
is_binary(TgtNode) ->
Resp = couch_util:with_db(DbName, fun(Db) ->
SrcUUID = couch_db:get_uuid(Db),
TgtUUID = couch_uuids:random(),
+ % See mem3_rep:purge_cp_body/3
CPDoc = #doc{
id = mem3_rep:make_purge_id(SrcUUID, TgtUUID),
body =
{[
- {<<"target_node">>, TgtNode},
- {<<"purge_seq">>, PurgeSeq}
+ {~"type", ~"internal_replication"},
+ {~"updated_on", os:system_time(second)},
+ {~"purge_seq", PurgeSeq},
+ {~"source", atom_to_binary(node())},
+ {~"target", TgtNode},
+ {~"range", [0, 16#ffffffff]}
]}
},
couch_db:update_docs(Db, [CPDoc], [])
end),
?assertMatch({ok, [_]}, Resp).
+compact(DbName) ->
+ {ok, _} = couch_util:with_db(DbName, fun couch_db:start_compact/1),
+ test_util:wait(fun() ->
+ case couch_util:with_db(DbName, fun couch_db:get_compactor_pid/1) of
+ Pid when is_pid(Pid) -> wait;
+ _ -> ok
+ end
+ end).
+
+purge_seqs(DbName) ->
+ couch_util:with_db(DbName, fun(Db) ->
+ {couch_db:get_oldest_purge_seq(Db), couch_db:get_purge_seq(Db)}
+ end).
+
rpc_update_doc(DbName, Doc) ->
{Pos, [Rev | _]} = Doc#doc.revs,
RROpt = {read_repair, [{tgt_node(), [{Pos, Rev}]}]},
diff --git a/src/mem3/src/mem3_rep.erl b/src/mem3/src/mem3_rep.erl
index 659adf275..c4acfac74 100644
--- a/src/mem3/src/mem3_rep.erl
+++ b/src/mem3/src/mem3_rep.erl
@@ -226,12 +226,21 @@ have_all_purge_checkpoints(Db) ->
have_all_purge_checkpoints(true, Db, [_ | _] = Shards) ->
ShardName = couch_db:name(Db),
UUID = couch_db:get_uuid(Db),
+ IsShardsDb = ShardName == mem3_sync:shards_db(),
FoldFun = fun(#doc{id = Id, body = {Props}}, Acc) ->
case Id of
<<?PURGE_PREFIX, UUID:?UUID_SIZE/binary, "-",
_:?UUID_SIZE/binary>> ->
case verify_checkpoint_shard(Shards, Props) of
true ->
- Range = couch_util:get_value(<<"range">>, Props),
+ Range =
+ case IsShardsDb of
+ % Shards db doesn't have a bona-fide range but
+ % we pretend it's a real db but giving it the
+ % full range to prevent it from being
+ % undefined.
+ true -> [0, ?RING_END];
+ false -> couch_util:get_value(<<"range">>,
Props)
+ end,
TBin = couch_util:get_value(<<"target">>, Props),
TNode = binary_to_existing_atom(TBin, latin1),
{ok, sets:add_element({TNode, Range}, Acc)};
@@ -244,7 +253,7 @@ have_all_purge_checkpoints(true, Db, [_ | _] = Shards) ->
end,
CheckpointSet = fold_purge_checkpoints(Db, FoldFun, couch_util:new_set()),
CopySet =
- case ShardName == mem3_sync:shards_db() of
+ case IsShardsDb of
true ->
% We're dealing with the shards db itself. By convention
% mem3:shards/1 returns a single #shard{} record with node =
@@ -543,7 +552,10 @@ pull_purges(Db, Count, #shard{} = SrcShard, #tgt{} = Tgt0,
HashFun) ->
end,
Infos1 = lists:filter(BelongsFun, Infos),
{ok, _} = couch_db:purge_docs(Db, Infos1, [?REPLICATED_CHANGES]),
- Body = purge_cp_body(SrcShard, TgtShard, ThroughSeq),
+ % A checkpoint from the target, but it's the same one the target
+ % uses when it pushes to us. So we write it from the target's point
+ % of view: it's the source and we are the target.
+ Body = purge_cp_body(TgtShard, SrcShard, ThroughSeq),
mem3_rpc:save_purge_checkpoint(TgtNode, TgtDbName, RemoteId, Body)
end,
Tgt#tgt{remaining = max(0, Remaining)}.
@@ -1443,9 +1455,12 @@ t_have_all_shards_db(_) ->
],
meck:expect(mem3, shards, 1, Shards),
- Src1 = #shard{name = Dbs, node = node(), range = Range},
- Tgt1 = #shard{name = Dbs, node = 'n2', range = Range},
- Tgt2 = #shard{name = Dbs, node = 'n3', range = Range},
+ % Same as in go/3, mem3_sync replicates by name, so its source and target
+ % shards don't have a range
+ Src1 = add_range(#shard{name = Dbs, node = node()}),
+ Tgt1 = add_range(#shard{name = Dbs, node = 'n2'}),
+ Tgt2 = add_range(#shard{name = Dbs, node = 'n3'}),
+ ?assertEqual(undefined, Src1#shard.range),
% We're the only node: don't expect any other checkpoints
meck:expect(mem3_sync, find_next_node, 0, node()),
diff --git a/src/mem3/test/eunit/mem3_rep_test.erl
b/src/mem3/test/eunit/mem3_rep_test.erl
index 321230ce4..0ecbf3d23 100644
--- a/src/mem3/test/eunit/mem3_rep_test.erl
+++ b/src/mem3/test/eunit/mem3_rep_test.erl
@@ -74,7 +74,8 @@ mem3_reshard_db_test_() ->
?TDEF_FE(replicate_with_partitions, ?TIMEOUT),
?TDEF_FE(replicate_to_and_from_local, ?TIMEOUT),
?TDEF_FE(replicate_with_purges, ?TIMEOUT),
- ?TDEF_FE(clean_purge_checkpoints, ?TIMEOUT)
+ ?TDEF_FE(clean_purge_checkpoints, ?TIMEOUT),
+ ?TDEF_FE(pull_purges_checkpoints, ?TIMEOUT)
]
}
}
@@ -287,6 +288,62 @@ clean_purge_checkpoints(#{allsrc := AllSrc, alltgt :=
AllTgt}) ->
?assertNot(is_map_key(Id3, LocalDocs)),
?assertNot(is_map_key(Id4, LocalDocs)).
+pull_purges_checkpoints(#{allsrc := AllSrc, alltgt := AllTgt}) ->
+ % Purge on the target, source has to pull purges both targets then
+ add_test_docs(AllTgt, #{docs => 10, purge => [0, 9]}),
+ [Src] = lists:sort(mem3:local_shards(AllSrc)),
+ [Tgt1, Tgt2] = lists:sort(mem3:local_shards(AllTgt)),
+ #shard{dbname = SrcDbName, range = SrcRange} = Src,
+ #shard{dbname = TgtDbName, range = R1} = Tgt1,
+ #shard{range = R2} = Tgt2,
+
+ % Fix up shard map as if source and target shards are copies of the same
+ % db. Then on each target the expected purge checkpoint is the same as the
+ % source
+ meck:expect(mem3, shards, fun(DbName) ->
+ case lists:member(DbName, [SrcDbName, TgtDbName]) of
+ true -> [Src, Tgt1, Tgt2];
+ false -> meck:passthrough([DbName])
+ end
+ end),
+ TgtNames = [Tgt1#shard.name, Tgt2#shard.name],
+ lists:foreach(
+ fun(TgtName) ->
+ % Targets should have some purges for source to pull
+ ?assert(purge_seq(TgtName) > 0),
+ ?assertNot(mem3_rep:have_all_purge_checkpoints(TgtName))
+ end,
+ TgtNames
+ ),
+
+ TMap = #{R1 => Tgt1, R2 => Tgt2},
+ Opts = [{batch_size, 1000}, {batch_count, all}],
+ ?assertMatch({ok, 0}, mem3_rep:go(Src, TMap, Opts)),
+ ?assertEqual(#{}, get_all_docs(AllSrc)),
+
+ % Source checkpoints pulls on the target. Those should look the same as if
+ % the target pushed to the source.
+ SrcUuid = uuid(Src#shard.name),
+ Node = atom_to_binary(Src#shard.node),
+ lists:foreach(
+ fun(TgtName) ->
+ TgtUuid = uuid(TgtName),
+ DocId = <<"_local/purge-mem3-", TgtUuid/binary, "-",
SrcUuid/binary>>,
+ #{DocId := Body} = local_docs(TgtName),
+ ?assertMatch(
+ #{
+ ~"source" := Node,
+ ~"target" := Node,
+ ~"range" := SrcRange,
+ ~"purge_seq" := PSeq
+ } when PSeq > 0,
+ Body
+ ),
+ ?assert(mem3_rep:have_all_purge_checkpoints(TgtName))
+ end,
+ TgtNames
+ ).
+
replicate_to_and_from_local(#{localdb := LocalDb, allsrc := ClusteredDb}) ->
% We'll just tests that we can pull purges from the target
add_test_docs(ClusteredDb, #{docs => 6, purge => [0, 2]}),
@@ -496,6 +553,12 @@ atts(Size) when is_integer(Size), Size >= 1 ->
])
].
+uuid(DbName) ->
+ couch_util:with_db(DbName, fun couch_db:get_uuid/1).
+
+purge_seq(DbName) ->
+ couch_util:with_db(DbName, fun couch_db:get_purge_seq/1).
+
local_docs(DbName) ->
{ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]),
FoldFun = fun(#doc{id = DocId, body = Body}, Acc) ->