This is an automated email from the ASF dual-hosted git repository. nickva pushed a commit to branch revpatch-fix in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit b4b2b2563d1e526e4ce6a9a1ecd8d01194bfe0de Author: Nick Vatamaniuc <[email protected]> AuthorDate: Thu Oct 15 12:32:39 2026 -0400 Fix revision patching for replicated changes Replicated revisions, which the target already has, may come in with various revision path lengths. This can easily happen if source endpoints have different revs_limits (see #6162). Previously, if the target had a VDU, updates went through `prep_and_validate_replicated_updates/5` which incorrectly dropped any revisions already in the tree as a no-op. However, those revision paths, even if the leafs were no-ops, could still patch-up or connect some existing revisions into a single revpath. To fix this, we have to let the revison paths only through to updater to do the revtree merge. We tag a document as a `revs_only` update (we have a precedent for adding meta tags to documents already) and drop its body and attachments, since we're intending to keep the existing disk value for its leaf always. In the updater if that revision still exists in the revtree we perform the rev merge and keep the disk (leaf) value as is. Fix: #6162 --- src/couch/src/couch_db.erl | 21 ++ src/couch/src/couch_db_updater.erl | 36 +++- .../test/eunit/couchdb_update_conflicts_tests.erl | 217 +++++++++++++++++++++ 3 files changed, 271 insertions(+), 3 deletions(-) diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl index e22f070b7..12e1bbbeb 100644 --- a/src/couch/src/couch_db.erl +++ b/src/couch/src/couch_db.erl @@ -1227,6 +1227,11 @@ prep_and_validate_replicated_updates( OldLeafsLU = [{Start, RevId} || {Start, [{RevId, _} | _]} <- OldLeafs], NewPaths = lists:map(fun couch_doc:to_path/1, Bucket), NewRevTree = couch_key_tree:multi_merge(OldTree, NewPaths), + % For any existing revision we only need to go to the updater if + % the new request may bring in rev path history we don't already + % have. But if we know the rev tree wouldn't change then we can + % skip that step. + RevTreeChanged = NewRevTree =/= OldTree, Leafs = couch_key_tree:get_all_leafs_full(NewRevTree), LeafRevsFullDict = dict:from_list([ {{Start, RevId}, FullPath} @@ -1268,6 +1273,16 @@ prep_and_validate_replicated_updates( Error -> {AccValidated, [{Doc, Error} | AccErrors2]} end; + _ when RevTreeChanged -> + % We already stored this revision or it's an + % internal node. So we don't validate. But that + % revision may bring in a longer connecting + % revision path that we don't already have (say + % if we had {1,[a]} and {2,[b]} but then get + % {2,[a,b]}). We'd want to the update to get + % just the new revpath so it can merge it with + % the body. + {[revs_only(Doc) | AccValidated], AccErrors2}; _ -> % this doc isn't a leaf or already exists in the tree. % ignore but consider it a success. @@ -1286,6 +1301,12 @@ prep_and_validate_replicated_updates( ) end. +% A revision we have. Updater will merges only its revpath. We drop its body +% adn attachments as the stored body is already on disk and attachment stubs +% don't need to be resolved. +revs_only(#doc{meta = Meta} = Doc) -> + Doc#doc{body = {[]}, atts = [], meta = [{revs_only, true} | Meta]}. + new_revid(#doc{body = Body, revs = {OldStart, OldRevs}, atts = Atts, deleted = Deleted}) -> DigestedAtts = lists:foldl( fun(Att, Acc) -> diff --git a/src/couch/src/couch_db_updater.erl b/src/couch/src/couch_db_updater.erl index 3229a2db9..140339e35 100644 --- a/src/couch/src/couch_db_updater.erl +++ b/src/couch/src/couch_db_updater.erl @@ -623,9 +623,39 @@ merge_rev_tree(_Db, OldInfo, NewDoc, _Client, true) -> % We're merging in revisions without caring about % conflicts. Most likely this is a replication update. OldTree = OldInfo#full_doc_info.rev_tree, - NewTree0 = couch_doc:to_path(NewDoc), - {NewTree, _} = couch_key_tree:merge(OldTree, NewTree0), - OldInfo#full_doc_info{rev_tree = NewTree}. + case replicated_path(OldTree, NewDoc) of + {ok, NewTree0} -> + {NewTree, _} = couch_key_tree:merge(OldTree, NewTree0), + OldInfo#full_doc_info{rev_tree = NewTree}; + ignore -> + OldInfo + end. + +% A revs_only doc is a revision we already have, but it was sent here since we may +% have to re-patch it's revision path. See prep_and_validate_replicated_updates/5 +replicated_path(OldTree, #doc{revs = {Pos, [RevId | _]}, meta = Meta} = Doc) -> + Path = couch_doc:to_path(Doc), + case lists:keymember(revs_only, 1, Meta) of + false -> + {ok, Path}; + true -> + case couch_key_tree:get_full_key_paths(OldTree, [{Pos, RevId}]) of + {[{Pos, [{RevId, Value} | _]} | _], []} -> + % The leaf Value we find in the old tree will be put in the + % leaf value when we merge. We're just patching the path + % here. + Fun = fun + (_Rev, _V, leaf) -> Value; + (_Rev, V, branch) -> V + end, + [Path1] = couch_key_tree:map(Fun, [Path]), + {ok, Path1}; + _ -> + % Rev is not in the tree any longer (might have been purged + % or an internal node of a longer path in the same request) + ignore + end + end. % Revpos were set before the document was grafted onto the deleted revision and % point to rev 1. Update them to the new revpos. This is needed when we diff --git a/src/couch/test/eunit/couchdb_update_conflicts_tests.erl b/src/couch/test/eunit/couchdb_update_conflicts_tests.erl index fc7884ed9..fddb126b1 100644 --- a/src/couch/test/eunit/couchdb_update_conflicts_tests.erl +++ b/src/couch/test/eunit/couchdb_update_conflicts_tests.erl @@ -20,6 +20,9 @@ -define(LOCAL_DOC_ID, <<"_local/foobar">>). -define(NUM_CLIENTS, [100, 500, 1000, 2000, 5000, 10000]). -define(TIMEOUT, 20000). +-define(VDU, ~"function(doc) {if (doc.balrog) {throw({forbidden:'shallnotpass'});}}"). +-define(ATT_NAME, ~"att.txt"). +-define(ATT_DATA, ~"original"). start() -> test_util:start_couch(). @@ -346,3 +349,217 @@ spawn_client(DbName, Doc) -> ok = couch_db:close(Db), exit(Result) end). + +% Test replicated changes coming in from sources with odd (non-default) revs +% limits. With a VDU we used to shortcut the prep-and-validate and not properly +% merge updated rev paths. +% +% Tests start with a two root tree that looks like: +% +% 1-a : 1-a came in from a revs limit = 1 source +% 2-b -> 3-c -> 4-d : 4-d came from a revs revs limit = 3 source +% +% Then we get 4-d again with a full revpath history of 1-a -> ... 4->d and then +% we have to check that we re-parent teh revision back to 1-a as a single path. + +replicated_mismatched_revslimit_test_() -> + { + "Replicated revs from mismatched revslimit sources", + { + setup, + fun start/0, + fun test_util:stop_couch/1, + { + foreach, + fun setup_db/0, + fun teardown_db/1, + [ + ?TDEF_FE(t_no_vdu), + ?TDEF_FE(t_with_vdu), + ?TDEF_FE(t_att_stub), + ?TDEF_FE(t_att_data), + ?TDEF_FE(t_no_rev_update), + ?TDEF_FE(t_internal_node_stemmed), + ?TDEF_FE(t_internal_node_full_path_patches_rev_tree), + ?TDEF_FE(t_multiple_revs_path_in_one_request) + ] + } + } + }. + +setup_db() -> + DbName = ?tempdb(), + {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]), + ok = couch_db:close(Db), + DbName. + +teardown_db(DbName) -> + ok = couch_server:delete(DbName, [?ADMIN_CTX]). + +t_no_vdu(DbName) -> + two_roots(DbName), + update(DbName, [d_4_full()]), + % leafs returns [{Pos, RevId, Deleted, RevPathLength}] + ?assertEqual([{4, ~"d", false, 4}], leafs(DbName)), + update(DbName, [e_5_deleted()]), + ?assertEqual([{5, ~"e", true, 5}], leafs(DbName)). + +t_with_vdu(DbName) -> + add_vdu(DbName), + two_roots(DbName), + % Patch the rev path but keep the old body on disk + update(DbName, [forbid(d_4_full())]), + % One rev path + ?assertEqual([{4, ~"d", false, 4}], leafs(DbName)), + % Old body + ?assertEqual({[{~"val", ~"d"}]}, body(DbName, {4, ~"d"})), + update(DbName, [e_5_deleted()]), + ?assertEqual([{5, ~"e", true, 5}], leafs(DbName)), + % New revs are validated, though + ?assertMatch({ok, [{_, {forbidden, _}}]}, update(DbName, [forbid(f_5_full())])). + +t_att_stub(DbName) -> + two_roots_with_att(DbName), + Stub = couch_att:new([ + {name, ?ATT_NAME}, + {data, stub}, + {type, ~"application/octet-stream"}, + {revpos, 4} + ]), + Doc = d_4_full(), + ?assertEqual({ok, []}, update(DbName, [Doc#doc{atts = [Stub]}])), + ?assertEqual([{4, ~"d", false, 4}], leafs(DbName)), + % Rev path patched but we kept the stored attachment + ?assertEqual(?ATT_DATA, attbin(DbName, {4, ~"d"})). + +t_att_data(DbName) -> + two_roots_with_att(DbName), + Att = att(~"replaced"), + Doc = d_4_full(), + update(DbName, [Doc#doc{atts = [Att]}]), + ?assertEqual([{4, ~"d", false, 4}], leafs(DbName)), + % Rev path patched but we kept the stored attachment + ?assertEqual(?ATT_DATA, attbin(DbName, {4, ~"d"})). + +t_no_rev_update(DbName) -> + add_vdu(DbName), + two_roots(DbName), + Seq = dbseq(DbName), + % 4-d with the history the db already has, seq doesn't change + update(DbName, [d_4_stemmed()]), + ?assertEqual(Seq, dbseq(DbName)), + ?assertEqual([{1, ~"a", false, 1}, {4, ~"d", false, 3}], leafs(DbName)). + +t_internal_node_stemmed(DbName) -> + add_vdu(DbName), + two_roots(DbName), + Seq = dbseq(DbName), + % 3-c is an internal rev. Sending its short path doesn't change anything + Doc = forbid(doc(3, [~"c", ~"b"], [{~"val", ~"c"}])), + update(DbName, [Doc]), + ?assertEqual(Seq, dbseq(DbName)), + % Sanity check + ?assertMatch([{{not_found, missing}, {3, ~"c"}}], revs(DbName, [{3, ~"c"}])), + % Rev tree doesn't change + ?assertEqual([{1, ~"a", false, 1}, {4, ~"d", false, 3}], leafs(DbName)). + +t_internal_node_full_path_patches_rev_tree(DbName) -> + add_vdu(DbName), + update(DbName, [d_4_stemmed()]), + % 3-c is an internal rev. We send it with a full path it should patch up + % the revs tree. Doesn't touch any leafs though. + Doc = forbid(doc(3, [~"c", ~"b", ~"a"], [{~"val", ~"c"}])), + update(DbName, [Doc]), + ?assertEqual([{4, ~"d", false, 4}], leafs(DbName)), + % Sanity check + ?assertMatch([{{not_found, missing}, {3, ~"c"}}], revs(DbName, [{3, ~"c"}])). + +t_multiple_revs_path_in_one_request(DbName) -> + add_vdu(DbName), + two_roots(DbName), + % 5-e is internal so it doesn't get validated. 6-f body gets stored. Both + % are not long enought to re-parent with 1-a + Doc1 = forbid(doc(5, [~"e", ~"d", ~"c"], [{~"val", ~"e"}])), + Doc2 = doc(6, [~"f", ~"e", ~"d", ~"c"], [{~"val", ~"f"}]), + ?assertEqual({ok, []}, update(DbName, [Doc1, Doc2])), + ?assertEqual([{1, ~"a", false, 1}, {6, ~"f", false, 5}], leafs(DbName)), + ?assertEqual({[{~"val", ~"f"}]}, body(DbName, {6, ~"f"})), + % 5-e internal never go through + ?assertMatch([{{not_found, missing}, {5, ~"e"}}], revs(DbName, [{5, ~"e"}])). + +two_roots(DbName) -> + update(DbName, [doc(1, [~"a"], [{~"val", ~"a"}])]), + update(DbName, [d_4_stemmed()]), + ?assertEqual([{1, ~"a", false, 1}, {4, ~"d", false, 3}], leafs(DbName)). + +two_roots_with_att(DbName) -> + update(DbName, [doc(1, [~"a"], [{~"val", ~"a"}])]), + Doc = d_4_stemmed(), + update(DbName, [Doc#doc{atts = [att(?ATT_DATA)]}]), + ?assertEqual([{1, ~"a", false, 1}, {4, ~"d", false, 3}], leafs(DbName)). + +d_4_stemmed() -> + doc(4, [~"d", ~"c", ~"b"], [{~"val", ~"d"}]). + +d_4_full() -> + doc(4, [~"d", ~"c", ~"b", ~"a"], [{~"val", ~"d"}]). + +e_5_deleted() -> + Doc = doc(5, [~"e", ~"d", ~"c"], []), + Doc#doc{deleted = true}. + +f_5_full() -> + doc(5, [~"f", ~"d", ~"c", ~"b", ~"a"], [{~"val", ~"f"}]). + +% This one will be rejected by the VDU +forbid(#doc{body = {Props}} = Doc) -> + Doc#doc{body = {Props ++ [{~"balrog", true}]}}. + +doc(Pos, Revs, Props) when is_list(Props) -> + #doc{id = ?DOC_ID, revs = {Pos, Revs}, body = {Props}}. + +att(Data) -> + couch_att:new([ + {name, ?ATT_NAME}, + {data, Data}, + {type, ~"application/octet-stream"}, + {att_len, byte_size(Data)}, + {disk_len, byte_size(Data)}, + {revpos, 4} + ]). + +add_vdu(DbName) -> + DDoc = #doc{id = ~"_design/nobalrog", body = {[{~"validate_doc_update", ?VDU}]}}, + couch_util:with_db(DbName, fun(Db) -> + {ok, _} = couch_db:update_doc(Db, DDoc, []) + end). + +update(DbName, Docs) -> + couch_util:with_db(DbName, fun(Db) -> + couch_db:update_docs(Db, Docs, [], ?REPLICATED_CHANGES) + end). + +% Get a list of {Pos, RevId, Deleted, RevPathLength} for all leafs +leafs(DbName) -> + couch_util:with_db(DbName, fun(Db) -> + {ok, Res} = couch_db:open_doc_revs(Db, ?DOC_ID, all, []), + RList = [{P, R, D, length(Rs)} || {ok, #doc{revs = {P, [R | _] = Rs}, deleted = D}} <- Res], + lists:sort(RList) + end). + +body(DbName, Rev) -> + [{ok, #doc{body = Body}}] = revs(DbName, [Rev]), + Body. + +attbin(DbName, Rev) -> + [{ok, #doc{atts = [Att]}}] = revs(DbName, [Rev]), + couch_att:to_binary(Att). + +revs(DbName, Revs) -> + couch_util:with_db(DbName, fun(Db) -> + {ok, Res} = couch_db:open_doc_revs(Db, ?DOC_ID, Revs, []), + Res + end). + +dbseq(DbName) -> + couch_util:with_db(DbName, fun couch_db:get_update_seq/1).
