This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new 9af4ae4c1 Fix revision patching for replicated changes
9af4ae4c1 is described below
commit 9af4ae4c1edeeba6b4b0bb6ec18f426117c55327
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Thu Oct 15 12:32:39 2026 -0400
Fix revision patching for replicated changes
Replicated revisions, which the target already has, may come in with various
revision path lengths. This can easily happen if source endpoints have
different revs_limits (see #6162). Previously, if the target had a VDU,
updates
went through `prep_and_validate_replicated_updates/5` which incorrectly
dropped
any revisions already in the tree as a no-op. However, those revision paths,
even if the leafs were no-ops, could still patch-up or connect some existing
revisions into a single revpath.
To fix this, we have to let the revison paths only through to updater to do
the
revtree merge. We tag a document as a `revs_only` update (we have a
precedent
for adding meta tags to documents already) and drop its body and
attachments,
since we're intending to keep the existing disk value for its leaf always.
In
the updater if that revision still exists in the revtree we perform the rev
merge and keep the disk (leaf) value as is.
Fix: #6162
---
src/couch/src/couch_db.erl | 12 ++
src/couch/src/couch_db_updater.erl | 35 +++-
.../test/eunit/couchdb_update_conflicts_tests.erl | 223 +++++++++++++++++++++
3 files changed, 267 insertions(+), 3 deletions(-)
diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl
index e22f070b7..131b58c96 100644
--- a/src/couch/src/couch_db.erl
+++ b/src/couch/src/couch_db.erl
@@ -1268,6 +1268,15 @@ prep_and_validate_replicated_updates(
Error ->
{AccValidated, [{Doc, Error} |
AccErrors2]}
end;
+ _ when NewRevTree =/= OldTree ->
+ % We already stored this revision or it's an
+ % internal node. So we don't validate. But that
+ % revision may bring in a longer connecting
+ % revision path that we don't already have (say
+ % if we had {1,[a]} and {2,[b]} but then get
+ % {2,[a,b]}). We'd want to the update to get
+ % new revpath {2, [a,b]} only.
+ {[revs_only(Doc) | AccValidated], AccErrors2};
_ ->
% this doc isn't a leaf or already exists in
the tree.
% ignore but consider it a success.
@@ -1286,6 +1295,9 @@ prep_and_validate_replicated_updates(
)
end.
+revs_only(#doc{meta = Meta} = Doc) ->
+ Doc#doc{body = {[]}, atts = [], meta = [{revs_only, true} | Meta]}.
+
new_revid(#doc{body = Body, revs = {OldStart, OldRevs}, atts = Atts, deleted =
Deleted}) ->
DigestedAtts = lists:foldl(
fun(Att, Acc) ->
diff --git a/src/couch/src/couch_db_updater.erl
b/src/couch/src/couch_db_updater.erl
index 3229a2db9..709f857b5 100644
--- a/src/couch/src/couch_db_updater.erl
+++ b/src/couch/src/couch_db_updater.erl
@@ -623,9 +623,38 @@ merge_rev_tree(_Db, OldInfo, NewDoc, _Client, true) ->
% We're merging in revisions without caring about
% conflicts. Most likely this is a replication update.
OldTree = OldInfo#full_doc_info.rev_tree,
- NewTree0 = couch_doc:to_path(NewDoc),
- {NewTree, _} = couch_key_tree:merge(OldTree, NewTree0),
- OldInfo#full_doc_info{rev_tree = NewTree}.
+ case replicated_path(OldTree, NewDoc) of
+ {ok, NewTree0} ->
+ {NewTree, _} = couch_key_tree:merge(OldTree, NewTree0),
+ OldInfo#full_doc_info{rev_tree = NewTree};
+ ignore ->
+ OldInfo
+ end.
+
+% A rev we already have keeps its leaf as stored (a body or nothing for an
+% internal node). Its path, though, is still merged, as it may patch the
+% revision history. A revs_only doc (see prep_and_validate for
+% replicated_updates) gets sent here without a body, so if its revision is not
+% in the tree any longer (it might have been purged, or is now an an internal
+% node of a longer path in the same request) there is nothing to store and it
+% is ignored, otherwise it's any revision not in the tree is a new revision.
+% revision not in the tree is new.
+replicated_path(OldTree, #doc{revs = {Pos, [RevId | _]}, meta = Meta} = Doc) ->
+ Path = couch_doc:to_path(Doc),
+ case couch_key_tree:get_full_key_paths(OldTree, [{Pos, RevId}]) of
+ {[{Pos, [{RevId, Value} | _]} | _], []} ->
+ Fun = fun
+ (_Rev, _V, leaf) -> Value;
+ (_Rev, V, branch) -> V
+ end,
+ [Path1] = couch_key_tree:map(Fun, [Path]),
+ {ok, Path1};
+ _ ->
+ case lists:keymember(revs_only, 1, Meta) of
+ true -> ignore;
+ false -> {ok, Path}
+ end
+ end.
% Revpos were set before the document was grafted onto the deleted revision and
% point to rev 1. Update them to the new revpos. This is needed when we
diff --git a/src/couch/test/eunit/couchdb_update_conflicts_tests.erl
b/src/couch/test/eunit/couchdb_update_conflicts_tests.erl
index fc7884ed9..199a79c14 100644
--- a/src/couch/test/eunit/couchdb_update_conflicts_tests.erl
+++ b/src/couch/test/eunit/couchdb_update_conflicts_tests.erl
@@ -20,6 +20,9 @@
-define(LOCAL_DOC_ID, <<"_local/foobar">>).
-define(NUM_CLIENTS, [100, 500, 1000, 2000, 5000, 10000]).
-define(TIMEOUT, 20000).
+-define(VDU, ~"function(doc) {if (doc.balrog)
{throw({forbidden:'shallnotpass'});}}").
+-define(ATT_NAME, ~"att.txt").
+-define(ATT_DATA, ~"original").
start() ->
test_util:start_couch().
@@ -346,3 +349,223 @@ spawn_client(DbName, Doc) ->
ok = couch_db:close(Db),
exit(Result)
end).
+
+% Test replicated changes coming in from sources with odd (non-default) revs
+% limits. With a VDU we used to shortcut the prep-and-validate and not properly
+% merge updated rev paths.
+%
+% Tests start with a two root tree that looks like:
+%
+% 1-a : 1-a came in from a revs limit = 1 source
+% 2-b -> 3-c -> 4-d : 4-d came from a revs revs limit = 3 source
+%
+% Then we get 4-d again with a full revpath history of 1-a -> ... 4->d and then
+% we have to check that we re-parent teh revision back to 1-a as a single path.
+
+replicated_mismatched_revslimit_test_() ->
+ {
+ "Replicated revs from mismatched revslimit sources",
+ {
+ setup,
+ fun start/0,
+ fun test_util:stop_couch/1,
+ {
+ foreach,
+ fun setup_db/0,
+ fun teardown_db/1,
+ [
+ ?TDEF_FE(t_no_vdu),
+ ?TDEF_FE(t_with_vdu),
+ ?TDEF_FE(t_no_vdu_keeps_body),
+ ?TDEF_FE(t_no_vdu_internal_node_stays_bodyless),
+ ?TDEF_FE(t_att_stub),
+ ?TDEF_FE(t_att_data),
+ ?TDEF_FE(t_internal_node_stemmed),
+ ?TDEF_FE(t_internal_node_full_path_patches_rev_tree),
+ ?TDEF_FE(t_multiple_revs_path_in_one_request)
+ ]
+ }
+ }
+ }.
+
+setup_db() ->
+ DbName = ?tempdb(),
+ {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
+ ok = couch_db:close(Db),
+ DbName.
+
+teardown_db(DbName) ->
+ ok = couch_server:delete(DbName, [?ADMIN_CTX]).
+
+t_no_vdu(DbName) ->
+ two_roots(DbName),
+ update(DbName, [d_4_full()]),
+ % leafs returns [{Pos, RevId, Deleted, RevPathLength}]
+ ?assertEqual([{4, ~"d", false, 4}], leafs(DbName)),
+ update(DbName, [e_5_deleted()]),
+ ?assertEqual([{5, ~"e", true, 5}], leafs(DbName)).
+
+t_no_vdu_keeps_body(DbName) ->
+ two_roots(DbName),
+ % 4-d again with a full path but a different body. The path is patched,
+ % the stored body stays.
+ update(DbName, [forbid(d_4_full())]),
+ ?assertEqual([{4, ~"d", false, 4}], leafs(DbName)),
+ ?assertEqual({[{~"val", ~"d"}]}, body(DbName, {4, ~"d"})).
+
+t_no_vdu_internal_node_stays_bodyless(DbName) ->
+ update(DbName, [d_4_stemmed()]),
+ % 3-c is an internal rev. Its full path patches the rev tree, but it
+ % doesn't get a body stored for it.
+ update(DbName, [doc(3, [~"c", ~"b", ~"a"], [{~"val", ~"c"}])]),
+ ?assertEqual([{4, ~"d", false, 4}], leafs(DbName)),
+ ?assertMatch([{{not_found, missing}, {3, ~"c"}}], revs(DbName, [{3,
~"c"}])),
+ % 1-a arriving later is an ancestor now, not a second root
+ update(DbName, [doc(1, [~"a"], [{~"val", ~"a"}])]),
+ ?assertEqual([{4, ~"d", false, 4}], leafs(DbName)).
+
+t_with_vdu(DbName) ->
+ vdu(DbName),
+ two_roots(DbName),
+ % Patch the rev path but keep the old body on disk
+ update(DbName, [forbid(d_4_full())]),
+ % One rev path
+ ?assertEqual([{4, ~"d", false, 4}], leafs(DbName)),
+ % Old body
+ ?assertEqual({[{~"val", ~"d"}]}, body(DbName, {4, ~"d"})),
+ update(DbName, [e_5_deleted()]),
+ ?assertEqual([{5, ~"e", true, 5}], leafs(DbName)),
+ % New revs are validated, though
+ ?assertMatch({ok, [{_, {forbidden, _}}]}, update(DbName,
[forbid(f_5_full())])).
+
+t_att_stub(DbName) ->
+ two_roots_with_att(DbName),
+ Stub = couch_att:new([
+ {name, ?ATT_NAME},
+ {data, stub},
+ {type, ~"application/octet-stream"},
+ {revpos, 4}
+ ]),
+ Doc = d_4_full(),
+ ?assertEqual({ok, []}, update(DbName, [Doc#doc{atts = [Stub]}])),
+ ?assertEqual([{4, ~"d", false, 4}], leafs(DbName)),
+ % Rev path patched but we kept the stored attachment
+ ?assertEqual(?ATT_DATA, attbin(DbName, {4, ~"d"})).
+
+t_att_data(DbName) ->
+ two_roots_with_att(DbName),
+ Att = att(~"replaced"),
+ Doc = d_4_full(),
+ update(DbName, [Doc#doc{atts = [Att]}]),
+ ?assertEqual([{4, ~"d", false, 4}], leafs(DbName)),
+ % Rev path patched but we kept the stored attachment
+ ?assertEqual(?ATT_DATA, attbin(DbName, {4, ~"d"})).
+
+t_internal_node_stemmed(DbName) ->
+ vdu(DbName),
+ two_roots(DbName),
+ % 3-c is an internal rev. Sending its short path doesn't change anything
+ Doc = forbid(doc(3, [~"c", ~"b"], [{~"val", ~"c"}])),
+ update(DbName, [Doc]),
+ % Sanity check
+ ?assertMatch([{{not_found, missing}, {3, ~"c"}}], revs(DbName, [{3,
~"c"}])),
+ % Rev tree doesn't change
+ ?assertEqual([{1, ~"a", false, 1}, {4, ~"d", false, 3}], leafs(DbName)).
+
+t_internal_node_full_path_patches_rev_tree(DbName) ->
+ vdu(DbName),
+ update(DbName, [d_4_stemmed()]),
+ % 3-c is an internal rev. We send it with a full path it should patch up
+ % the revs tree. Doesn't touch any leafs though.
+ Doc = forbid(doc(3, [~"c", ~"b", ~"a"], [{~"val", ~"c"}])),
+ update(DbName, [Doc]),
+ ?assertEqual([{4, ~"d", false, 4}], leafs(DbName)),
+ % Sanity check
+ ?assertMatch([{{not_found, missing}, {3, ~"c"}}], revs(DbName, [{3,
~"c"}])).
+
+t_multiple_revs_path_in_one_request(DbName) ->
+ vdu(DbName),
+ two_roots(DbName),
+ % 5-e is internal so it doesn't get validated. 6-f body gets stored. Both
+ % are not long enought to re-parent with 1-a
+ Doc1 = forbid(doc(5, [~"e", ~"d", ~"c"], [{~"val", ~"e"}])),
+ Doc2 = doc(6, [~"f", ~"e", ~"d", ~"c"], [{~"val", ~"f"}]),
+ ?assertEqual({ok, []}, update(DbName, [Doc1, Doc2])),
+ ?assertEqual([{1, ~"a", false, 1}, {6, ~"f", false, 5}], leafs(DbName)),
+ ?assertEqual({[{~"val", ~"f"}]}, body(DbName, {6, ~"f"})),
+ % 5-e internal never go through
+ ?assertMatch([{{not_found, missing}, {5, ~"e"}}], revs(DbName, [{5,
~"e"}])).
+
+two_roots(DbName) ->
+ update(DbName, [doc(1, [~"a"], [{~"val", ~"a"}])]),
+ update(DbName, [d_4_stemmed()]),
+ ?assertEqual([{1, ~"a", false, 1}, {4, ~"d", false, 3}], leafs(DbName)).
+
+two_roots_with_att(DbName) ->
+ update(DbName, [doc(1, [~"a"], [{~"val", ~"a"}])]),
+ Doc = d_4_stemmed(),
+ update(DbName, [Doc#doc{atts = [att(?ATT_DATA)]}]),
+ ?assertEqual([{1, ~"a", false, 1}, {4, ~"d", false, 3}], leafs(DbName)).
+
+d_4_stemmed() ->
+ doc(4, [~"d", ~"c", ~"b"], [{~"val", ~"d"}]).
+
+d_4_full() ->
+ doc(4, [~"d", ~"c", ~"b", ~"a"], [{~"val", ~"d"}]).
+
+e_5_deleted() ->
+ Doc = doc(5, [~"e", ~"d", ~"c"], []),
+ Doc#doc{deleted = true}.
+
+f_5_full() ->
+ doc(5, [~"f", ~"d", ~"c", ~"b", ~"a"], [{~"val", ~"f"}]).
+
+% This one will be rejected by the VDU
+forbid(#doc{body = {Props}} = Doc) ->
+ Doc#doc{body = {Props ++ [{~"balrog", true}]}}.
+
+doc(Pos, Revs, Props) when is_list(Props) ->
+ #doc{id = ?DOC_ID, revs = {Pos, Revs}, body = {Props}}.
+
+att(Data) ->
+ couch_att:new([
+ {name, ?ATT_NAME},
+ {data, Data},
+ {type, ~"application/octet-stream"},
+ {att_len, byte_size(Data)},
+ {disk_len, byte_size(Data)},
+ {revpos, 4}
+ ]).
+
+body(DbName, Rev) ->
+ [{ok, #doc{body = Body}}] = revs(DbName, [Rev]),
+ Body.
+
+attbin(DbName, Rev) ->
+ [{ok, #doc{atts = [Att]}}] = revs(DbName, [Rev]),
+ couch_att:to_binary(Att).
+
+revs(DbName, Revs) ->
+ couch_util:with_db(DbName, fun(Db) ->
+ {ok, Res} = couch_db:open_doc_revs(Db, ?DOC_ID, Revs, []),
+ Res
+ end).
+
+vdu(DbName) ->
+ DDoc = #doc{id = ~"_design/nobalrog", body = {[{~"validate_doc_update",
?VDU}]}},
+ couch_util:with_db(DbName, fun(Db) ->
+ {ok, _} = couch_db:update_doc(Db, DDoc, [])
+ end).
+
+update(DbName, Docs) ->
+ couch_util:with_db(DbName, fun(Db) ->
+ couch_db:update_docs(Db, Docs, [], ?REPLICATED_CHANGES)
+ end).
+
+% Get a list of {Pos, RevId, Deleted, RevPathLength} for all leafs
+leafs(DbName) ->
+ couch_util:with_db(DbName, fun(Db) ->
+ {ok, Res} = couch_db:open_doc_revs(Db, ?DOC_ID, all, []),
+ RList = [{P, R, D, length(Rs)} || {ok, #doc{revs = {P, [R | _] = Rs},
deleted = D}} <- Res],
+ lists:sort(RList)
+ end).