This is an automated email from the ASF dual-hosted git repository. nickva pushed a commit to branch fix-pull-purge-checkpoints in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 23af37f903a65633c379eea5e55698468c176db8 Author: Nick Vatamaniuc <[email protected]> AuthorDate: Sat Nov 7 18:40:04 2026 -0500 Fix stuck mem3 purge checkpoints Observed a few rare cases of purges being stuck in production We had the wrong field name in read repair purge checkpoint filter. Before we looked up `target_node` in the purge checkpoint docs but it should really be `target` (see mem3_rep:purge_cp_body/3). This way every node started at purge seq 0. `pull_purges/5` had the target/source swapped. The checkpoint lives on the target so we should always write it from the target's perspective, meaning the target is the source and we're the target. There was an off-by-one error in `have_all_purge_checkpoints/1` Shards db checkpoints have an undefined range (since they are effectively local dbs replicated in special ring topology) and they replicated by name. As such they never mathed the `{Node, [0, ?RING_END]}` copies so have_all_purge_checkpoints/1 was always false for them. Fortunately this is easy to fix, and just deleting the checkpoints and letting the next internal replicaton cycle start over from the oldest purge seq works. --- src/couch/src/couch_bt_engine_compactor.erl | 5 +- .../test/eunit/couch_bt_engine_compactor_tests.erl | 33 ++++++++++- src/fabric/src/fabric_rpc.erl | 12 +++- src/fabric/test/eunit/fabric_rpc_purge_tests.erl | 51 ++++++++++++++++- src/mem3/src/mem3_rep.erl | 27 +++++++-- src/mem3/test/eunit/mem3_rep_test.erl | 65 +++++++++++++++++++++- 6 files changed, 179 insertions(+), 14 deletions(-) diff --git a/src/couch/src/couch_bt_engine_compactor.erl b/src/couch/src/couch_bt_engine_compactor.erl index 36c149217..2a8de05cf 100644 --- a/src/couch/src/couch_bt_engine_compactor.erl +++ b/src/couch/src/couch_bt_engine_compactor.erl @@ -176,9 +176,12 @@ copy_purge_info(#comp_st{} = CompSt) -> % that case we'd end up with a "hole" between 500 and 1001 -- a new % minimum purge sequence of 500, but the oldest checkpoint is would be % 1001. + % + % Keep infos with PSeq > MinPurgeSeq. To get all of them, including the + % oldest need to use Oldest-1. case mem3_rep:have_all_purge_checkpoints(Db) of true -> couch_db:get_minimum_purge_seq(Db); - false -> couch_db:get_oldest_purge_seq(Db) + false -> max(0, couch_db:get_oldest_purge_seq(Db) - 1) end end), OldPSTree = OldSt#st.purge_seq_tree, diff --git a/src/couch/test/eunit/couch_bt_engine_compactor_tests.erl b/src/couch/test/eunit/couch_bt_engine_compactor_tests.erl index 63df30fb0..69667a7e4 100644 --- a/src/couch/test/eunit/couch_bt_engine_compactor_tests.erl +++ b/src/couch/test/eunit/couch_bt_engine_compactor_tests.erl @@ -41,7 +41,8 @@ basic_compaction_test_() -> fun teardown/1, [ fun compaction_resume/1, - fun is_compacting_works/1 + fun is_compacting_works/1, + fun preserve_purges_without_all_checkpoints/1 ] } }. @@ -81,6 +82,36 @@ is_compacting_works(DbName) -> meck:unload(couch_emsort) end). +preserve_purges_without_all_checkpoints(DbName) -> + ?_test(begin + purge_docs(DbName), + ?assertEqual({1, 3, 3}, purge_info_stats(DbName)), + meck:new(mem3_rep, [passthrough]), + meck:expect(mem3_rep, have_all_purge_checkpoints, 1, false), + compact_db(DbName), + ?assert(meck:called(mem3_rep, have_all_purge_checkpoints, 1)), + ?assertEqual({1, 3, 3}, purge_info_stats(DbName)), + meck:unload(mem3_rep) + end). + +purge_docs(DbName) -> + couch_util:with_db(DbName, fun(Db) -> + PurgeFun = fun(DocId) -> + {ok, #doc{revs = {Pos, [Rev | _]}}} = couch_db:open_doc(Db, DocId, []), + {couch_uuids:random(), DocId, [{Pos, Rev}]} + end, + PInfos = lists:map(PurgeFun, [~"doc1", ~"doc2", ~"doc3"]), + {ok, _} = couch_db:purge_docs(Db, PInfos) + end). + +% Return {OldestPurgeSeq, PurgeSeq, PurgeInfosCount} +purge_info_stats(DbName) -> + couch_util:with_db(DbName, fun(Db) -> + FoldFun = fun(_, Acc) -> {ok, Acc + 1} end, + {ok, Count} = couch_db:fold_purge_infos(Db, FoldFun, 0), + {couch_db:get_oldest_purge_seq(Db), couch_db:get_purge_seq(Db), Count} + end). + check_db_validity(DbName) -> couch_util:with_db(DbName, fun(Db) -> ?assertEqual({ok, 3}, couch_db:get_doc_count(Db)), diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl index a05d59f96..1a2bc4928 100644 --- a/src/fabric/src/fabric_rpc.erl +++ b/src/fabric/src/fabric_rpc.erl @@ -582,14 +582,18 @@ read_repair_filter(Db, Docs, NodeRevs) -> NodeSeqs = get_node_seqs(Db, Nodes), DbPSeq = couch_db:get_purge_seq(Db), + OldestPSeq = couch_db:get_oldest_purge_seq(Db), Lag = config:get_integer("couchdb", "read_repair_lag", 100), % Filter out read-repair updates from any node that is % so out of date that it would force us to scan a large - % number of purge infos + % number of purge infos. And drop nodes behind our oldest purge info. + % Compaction might already removed infos they haven't seen, so we can't tell if + % their revisions were purged, and fold_purge_infos/4 would + % fail with an invalid_start_purge_seq error. NodeFiltFun = fun({Node, _Revs}) -> {Node, NodeSeq} = lists:keyfind(Node, 1, NodeSeqs), - NodeSeq >= DbPSeq - Lag + NodeSeq >= DbPSeq - Lag andalso NodeSeq >= OldestPSeq - 1 end, RecentNodeRevs = lists:filter(NodeFiltFun, NonLocalNodeRevs), @@ -625,7 +629,9 @@ get_node_seqs(Db, Nodes) -> FoldFun = fun(#doc{id = Id, body = {Props}}, Acc) -> case Id of <<?LOCAL_DOC_PREFIX, "purge-mem3-", _/binary>> -> - TgtNode = couch_util:get_value(<<"target_node">>, Props), + % Should match mem3_rep:purge_cp_body/3. There "target" + % is the shard we replicate our purges TO. + TgtNode = couch_util:get_value(~"target", Props), PurgeSeq = couch_util:get_value(<<"purge_seq">>, Props), case lists:keyfind(TgtNode, 1, Acc) of {_, OldSeq} -> diff --git a/src/fabric/test/eunit/fabric_rpc_purge_tests.erl b/src/fabric/test/eunit/fabric_rpc_purge_tests.erl index fa6123dfc..3d2eb35b2 100644 --- a/src/fabric/test/eunit/fabric_rpc_purge_tests.erl +++ b/src/fabric/test/eunit/fabric_rpc_purge_tests.erl @@ -60,6 +60,19 @@ main_test_() -> ?TDEF_FE(t_no_filter_different_node), ?TDEF_FE(t_no_filter_after_repl) ] + }, + { + foreach, + fun setup_trimmed_purge/0, + fun teardown_trimmed_purge/1, + [ + ?TDEF_FE(t_filter), + ?TDEF_FE(t_filter_unknown_node), + ?TDEF_FE(t_filter_local_node), + ?TDEF_FE(t_no_filter_old_node), + ?TDEF_FE(t_no_filter_different_node), + ?TDEF_FE(t_no_filter_after_repl) + ] } ] }. @@ -104,6 +117,21 @@ setup_multi_purge() -> teardown_multi_purge(Ctx) -> teardown_single_purge(Ctx). +% Like the multi case above, but with the compactor trimmed oldest purge infos +% 1 to 3. After that the oldest purge seq is 4. Nodes with checkpoints before 3 +% are too far behind and their read repair updates should be filtered out. +setup_trimmed_purge() -> + {DbName, _, _, _} = Ctx = setup_multi_purge(), + couch_util:with_db(DbName, fun(Db) -> + ok = couch_db:set_purge_infos_limit(Db, 2) + end), + compact(DbName), + ?assertEqual({4, 5}, purge_seqs(DbName)), + Ctx. + +teardown_trimmed_purge(Ctx) -> + teardown_single_purge(Ctx). + t_no_purge_no_filter(DbName) -> DocId = <<"0003">>, @@ -239,18 +267,37 @@ create_purge_checkpoint(DbName, PurgeSeq, TgtNode) when is_binary(TgtNode) -> Resp = couch_util:with_db(DbName, fun(Db) -> SrcUUID = couch_db:get_uuid(Db), TgtUUID = couch_uuids:random(), + % See mem3_rep:purge_cp_body/3 CPDoc = #doc{ id = mem3_rep:make_purge_id(SrcUUID, TgtUUID), body = {[ - {<<"target_node">>, TgtNode}, - {<<"purge_seq">>, PurgeSeq} + {~"type", ~"internal_replication"}, + {~"updated_on", os:system_time(second)}, + {~"purge_seq", PurgeSeq}, + {~"source", atom_to_binary(node())}, + {~"target", TgtNode}, + {~"range", [0, 16#ffffffff]} ]} }, couch_db:update_docs(Db, [CPDoc], []) end), ?assertMatch({ok, [_]}, Resp). +compact(DbName) -> + {ok, _} = couch_util:with_db(DbName, fun couch_db:start_compact/1), + test_util:wait(fun() -> + case couch_util:with_db(DbName, fun couch_db:get_compactor_pid/1) of + Pid when is_pid(Pid) -> wait; + _ -> ok + end + end). + +purge_seqs(DbName) -> + couch_util:with_db(DbName, fun(Db) -> + {couch_db:get_oldest_purge_seq(Db), couch_db:get_purge_seq(Db)} + end). + rpc_update_doc(DbName, Doc) -> {Pos, [Rev | _]} = Doc#doc.revs, RROpt = {read_repair, [{tgt_node(), [{Pos, Rev}]}]}, diff --git a/src/mem3/src/mem3_rep.erl b/src/mem3/src/mem3_rep.erl index 659adf275..d622a0f56 100644 --- a/src/mem3/src/mem3_rep.erl +++ b/src/mem3/src/mem3_rep.erl @@ -226,12 +226,21 @@ have_all_purge_checkpoints(Db) -> have_all_purge_checkpoints(true, Db, [_ | _] = Shards) -> ShardName = couch_db:name(Db), UUID = couch_db:get_uuid(Db), + IsShardsDb = ShardName == mem3_sync:shards_db(), FoldFun = fun(#doc{id = Id, body = {Props}}, Acc) -> case Id of <<?PURGE_PREFIX, UUID:?UUID_SIZE/binary, "-", _:?UUID_SIZE/binary>> -> case verify_checkpoint_shard(Shards, Props) of true -> - Range = couch_util:get_value(<<"range">>, Props), + Range = + case IsShardsDb of + % Shards db doesn't have a bona-fide range but + % we pretend it's a real db but givig it the + % full range to prevent it from being + % undefined. + true -> [0, ?RING_END]; + false -> couch_util:get_value(<<"range">>, Props) + end, TBin = couch_util:get_value(<<"target">>, Props), TNode = binary_to_existing_atom(TBin, latin1), {ok, sets:add_element({TNode, Range}, Acc)}; @@ -244,7 +253,7 @@ have_all_purge_checkpoints(true, Db, [_ | _] = Shards) -> end, CheckpointSet = fold_purge_checkpoints(Db, FoldFun, couch_util:new_set()), CopySet = - case ShardName == mem3_sync:shards_db() of + case IsShardsDb of true -> % We're dealing with the shards db itself. By convention % mem3:shards/1 returns a single #shard{} record with node = @@ -543,7 +552,10 @@ pull_purges(Db, Count, #shard{} = SrcShard, #tgt{} = Tgt0, HashFun) -> end, Infos1 = lists:filter(BelongsFun, Infos), {ok, _} = couch_db:purge_docs(Db, Infos1, [?REPLICATED_CHANGES]), - Body = purge_cp_body(SrcShard, TgtShard, ThroughSeq), + % A checkpoint form the target but it's the same one the target + % uses when it pushes to us. So we write it from the target's point + % of view: it's the source and we are the target. + Body = purge_cp_body(TgtShard, SrcShard, ThroughSeq), mem3_rpc:save_purge_checkpoint(TgtNode, TgtDbName, RemoteId, Body) end, Tgt#tgt{remaining = max(0, Remaining)}. @@ -1443,9 +1455,12 @@ t_have_all_shards_db(_) -> ], meck:expect(mem3, shards, 1, Shards), - Src1 = #shard{name = Dbs, node = node(), range = Range}, - Tgt1 = #shard{name = Dbs, node = 'n2', range = Range}, - Tgt2 = #shard{name = Dbs, node = 'n3', range = Range}, + % Same asin go/3, mem3_sync replicates by name, so its source and target + % shards don't have a range + Src1 = add_range(#shard{name = Dbs, node = node()}), + Tgt1 = add_range(#shard{name = Dbs, node = 'n2'}), + Tgt2 = add_range(#shard{name = Dbs, node = 'n3'}), + ?assertEqual(undefined, Src1#shard.range), % We're the only node: don't expect any other checkpoints meck:expect(mem3_sync, find_next_node, 0, node()), diff --git a/src/mem3/test/eunit/mem3_rep_test.erl b/src/mem3/test/eunit/mem3_rep_test.erl index 321230ce4..ec06d90ec 100644 --- a/src/mem3/test/eunit/mem3_rep_test.erl +++ b/src/mem3/test/eunit/mem3_rep_test.erl @@ -74,7 +74,8 @@ mem3_reshard_db_test_() -> ?TDEF_FE(replicate_with_partitions, ?TIMEOUT), ?TDEF_FE(replicate_to_and_from_local, ?TIMEOUT), ?TDEF_FE(replicate_with_purges, ?TIMEOUT), - ?TDEF_FE(clean_purge_checkpoints, ?TIMEOUT) + ?TDEF_FE(clean_purge_checkpoints, ?TIMEOUT), + ?TDEF_FE(pull_purges_checkpoints, ?TIMEOUT) ] } } @@ -287,6 +288,62 @@ clean_purge_checkpoints(#{allsrc := AllSrc, alltgt := AllTgt}) -> ?assertNot(is_map_key(Id3, LocalDocs)), ?assertNot(is_map_key(Id4, LocalDocs)). +pull_purges_checkpoints(#{allsrc := AllSrc, alltgt := AllTgt}) -> + % Purge on the target, source has to pull purges both targets hten + add_test_docs(AllTgt, #{docs => 10, purge => [0, 9]}), + [Src] = lists:sort(mem3:local_shards(AllSrc)), + [Tgt1, Tgt2] = lists:sort(mem3:local_shards(AllTgt)), + #shard{dbname = SrcDbName, range = SrcRange} = Src, + #shard{dbname = TgtDbName, range = R1} = Tgt1, + #shard{range = R2} = Tgt2, + + % Fix up shard map as if source and target shards are copies of the same + % db. Then on each target the expected purge checkpoint is the same as the + % source + meck:expect(mem3, shards, fun(DbName) -> + case lists:member(DbName, [SrcDbName, TgtDbName]) of + true -> [Src, Tgt1, Tgt2]; + false -> meck:passthrough([DbName]) + end + end), + TgtNames = [Tgt1#shard.name, Tgt2#shard.name], + lists:foreach( + fun(TgtName) -> + % Targets should have some purges for source to pull + ?assert(purge_seq(TgtName) > 0), + ?assertNot(mem3_rep:have_all_purge_checkpoints(TgtName)) + end, + TgtNames + ), + + TMap = #{R1 => Tgt1, R2 => Tgt2}, + Opts = [{batch_size, 1000}, {batch_count, all}], + ?assertMatch({ok, 0}, mem3_rep:go(Src, TMap, Opts)), + ?assertEqual(#{}, get_all_docs(AllSrc)), + + % Source checkpoints pulls on the target. Those should look the same as if + % the target pushed to the source. + SrcUuid = uuid(Src#shard.name), + Node = atom_to_binary(Src#shard.node), + lists:foreach( + fun(TgtName) -> + TgtUuid = uuid(TgtName), + DocId = <<"_local/purge-mem3-", TgtUuid/binary, "-", SrcUuid/binary>>, + #{DocId := Body} = local_docs(TgtName), + ?assertMatch( + #{ + ~"source" := Node, + ~"target" := Node, + ~"range" := SrcRange, + ~"purge_seq" := PSeq + } when PSeq > 0, + Body + ), + ?assert(mem3_rep:have_all_purge_checkpoints(TgtName)) + end, + TgtNames + ). + replicate_to_and_from_local(#{localdb := LocalDb, allsrc := ClusteredDb}) -> % We'll just tests that we can pull purges from the target add_test_docs(ClusteredDb, #{docs => 6, purge => [0, 2]}), @@ -496,6 +553,12 @@ atts(Size) when is_integer(Size), Size >= 1 -> ]) ]. +uuid(DbName) -> + couch_util:with_db(DbName, fun couch_db:get_uuid/1). + +purge_seq(DbName) -> + couch_util:with_db(DbName, fun couch_db:get_purge_seq/1). + local_docs(DbName) -> {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]), FoldFun = fun(#doc{id = DocId, body = Body}, Acc) ->
