This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new 83a487b75 Fix attachment revisions when recreating a deleted doc
83a487b75 is described below
commit 83a487b754af09d271ce44a60fae6e4f4c24aa8b
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Thu Sep 24 01:28:56 2026 -0400
Fix attachment revisions when recreating a deleted doc
When we re-create a deleted doc, we forgot to update the revpos of the
attachments. They were left at 1- when they should be at 3-
Fixes: https://github.com/apache/couchdb/issues/6126
---
src/couch/src/couch_att.erl | 49 +++++-
src/couch/src/couch_db_updater.erl | 48 +++++-
.../test/eunit/couch_db_recreate_att_tests.erl | 184 +++++++++++++++++++++
3 files changed, 271 insertions(+), 10 deletions(-)
diff --git a/src/couch/src/couch_att.erl b/src/couch/src/couch_att.erl
index c7e051a28..4c3678b5f 100644
--- a/src/couch/src/couch_att.erl
+++ b/src/couch/src/couch_att.erl
@@ -29,7 +29,9 @@
-export([
size_info/1,
to_disk_term/1,
- from_disk_term/2
+ from_disk_term/2,
+ disk_term_set_revpos/2,
+ stub_from_disk_term/1
]).
-export([
@@ -365,6 +367,28 @@ to_disk_term(Att) ->
),
{list_to_tuple(lists:reverse(Base)), Extended}.
+%% Set revpos in a disk term
+disk_term_set_revpos({Base, Extended}, RevPos) when is_tuple(Base),
is_list(Extended) ->
+ {disk_term_set_revpos(Base, RevPos), Extended};
+disk_term_set_revpos({_Name, _Type, _Sp, _AttLen, _DiskLen, _RevPos, _Md5,
_Enc} = Term, RevPos) ->
+ setelement(6, Term, RevPos).
+
+%% Make a stub from a disk term the same way as to_disk_term/1. It has
everything
+%% except the data so we don't need to open a stream here
+stub_from_disk_term({Base, Extended}) when is_tuple(Base), is_list(Extended) ->
+ store(Extended, stub_from_disk_term(Base));
+stub_from_disk_term({Name, Type, _Sp, AttLen, DiskLen, RevPos, Md5, Enc}) ->
+ #att{
+ name = Name,
+ type = Type,
+ att_len = AttLen,
+ disk_len = DiskLen,
+ md5 = Md5,
+ revpos = RevPos,
+ data = stub,
+ encoding = upgrade_encoding(Enc)
+ }.
+
%% The new disk term format is a simple wrapper around the legacy format. Base
%% properties will remain in a tuple while the new fields and possibly data
from
%% future extensions will be stored in a list of atom/value pairs. While this
is
@@ -897,6 +921,29 @@ attachment_stub_merge_test_() ->
%% attachment errors.
{"Attachment stub merging tests", []}.
+disk_term_set_revpos_test() ->
+ {Base, Ext} = disk_term_set_revpos({{~"a", t, sp, 1, 1, 1, m, e}, []}, 5),
+ ?assertEqual({~"a", t, sp, 1, 1, 5, m, e}, Base),
+ ?assertEqual([], Ext),
+ Term = disk_term_set_revpos({~"a", t, sp, 1, 1, 1, m, e}, 7),
+ ?assertEqual(7, element(6, Term)).
+
+stub_from_disk_term_test() ->
+ Att = new([
+ {name, ~"a.txt"},
+ {type, ~"text/plain"},
+ {att_len, 3},
+ {disk_len, 5},
+ {md5, ~"md5"},
+ {revpos, 2},
+ {data, {stream, {couch_bt_engine_stream, {fake_fd, fake_sp}}}},
+ {encoding, gzip}
+ ]),
+ ?assertEqual(Att#att{data = stub}, stub_from_disk_term(to_disk_term(Att))),
+ ExtendedAtt = store(headers, [{~"X-Foo", ~"bar"}], Att),
+ ExtendedStub = store(data, stub, ExtendedAtt),
+ ?assertEqual(ExtendedStub, stub_from_disk_term(to_disk_term(ExtendedAtt))).
+
%% Test generators
test_non_upgrading_fields(Attachment) ->
diff --git a/src/couch/src/couch_db_updater.erl
b/src/couch/src/couch_db_updater.erl
index 8e697039f..3229a2db9 100644
--- a/src/couch/src/couch_db_updater.erl
+++ b/src/couch/src/couch_db_updater.erl
@@ -28,7 +28,8 @@
add_infos = [],
rem_seqs = [],
cur_seq,
- full_partitions = []
+ full_partitions = [],
+ db
}).
init({Engine, DbName, FilePath, Options0}) ->
@@ -474,14 +475,15 @@ merge_rev_trees([NewDocs | RestDocsList], [OldDocInfo |
RestOldInfo], Acc) ->
#merge_acc{
revs_limit = Limit,
replicated_changes = ReplicatedChanges,
- full_partitions = FullPartitions
+ full_partitions = FullPartitions,
+ db = Db
} = Acc,
% Track doc ids so we can debug large revision trees
erlang:put(last_id_merged, OldDocInfo#full_doc_info.id),
NewDocInfo0 = lists:foldl(
fun({Client, NewDoc}, OldInfoAcc) ->
- NewInfo = merge_rev_tree(OldInfoAcc, NewDoc, Client,
ReplicatedChanges),
+ NewInfo = merge_rev_tree(Db, OldInfoAcc, NewDoc, Client,
ReplicatedChanges),
case is_overflowed(NewInfo, OldInfoAcc, FullPartitions) of
true when not ReplicatedChanges ->
DocId = NewInfo#full_doc_info.id,
@@ -535,7 +537,7 @@ merge_rev_trees([NewDocs | RestDocsList], [OldDocInfo |
RestOldInfo], Acc) ->
merge_rev_trees(RestDocsList, RestOldInfo, NewAcc)
end.
-merge_rev_tree(OldInfo, NewDoc, Client, false) when
+merge_rev_tree(Db, OldInfo, NewDoc, Client, false) when
OldInfo#full_doc_info.deleted
->
% We're recreating a document that was previously
@@ -561,12 +563,17 @@ merge_rev_tree(OldInfo, NewDoc, Client, false) when
_ ->
NewDoc#doc.body
end,
+ % Attachments here are disk terms. new_revid/1 can't read them, it
+ % wants stubs. So we'll give it stubs with the right digests.
+ Atts = [couch_att:stub_from_disk_term(Att) || Att <-
NewDoc#doc.atts],
RevIdDoc = NewDoc#doc{
revs = {OldPos, [OldRev]},
- body = Body
+ body = Body,
+ atts = Atts
},
NewRevId = couch_db:new_revid(RevIdDoc),
- NewDoc2 = NewDoc#doc{revs = {OldPos + 1, [NewRevId, OldRev]}},
+ NewDoc1 = set_recreated_att_revpos(Db, NewDoc, OldPos + 1),
+ NewDoc2 = NewDoc1#doc{revs = {OldPos + 1, [NewRevId, OldRev]}},
% Merge our modified new doc into the tree
#full_doc_info{rev_tree = OldTree} = OldInfo,
@@ -586,7 +593,7 @@ merge_rev_tree(OldInfo, NewDoc, Client, false) when
send_result(Client, NewDoc, conflict),
OldInfo
end;
-merge_rev_tree(OldInfo, NewDoc, Client, false) ->
+merge_rev_tree(_Db, OldInfo, NewDoc, Client, false) ->
% We're attempting to merge a new revision into an
% undeleted document. To not be a conflict we require
% that the merge results in extending a branch.
@@ -612,7 +619,7 @@ merge_rev_tree(OldInfo, NewDoc, Client, false) ->
send_result(Client, NewDoc, conflict),
OldInfo
end;
-merge_rev_tree(OldInfo, NewDoc, _Client, true) ->
+merge_rev_tree(_Db, OldInfo, NewDoc, _Client, true) ->
% We're merging in revisions without caring about
% conflicts. Most likely this is a replication update.
OldTree = OldInfo#full_doc_info.rev_tree,
@@ -620,6 +627,28 @@ merge_rev_tree(OldInfo, NewDoc, _Client, true) ->
{NewTree, _} = couch_key_tree:merge(OldTree, NewTree0),
OldInfo#full_doc_info{rev_tree = NewTree}.
+% Revpos were set before the document was grafted onto the deleted revision and
+% point to rev 1. Update them to the new revpos. This is needed when we
+% re-create a document which was previously deleted.
+set_recreated_att_revpos(_Db, #doc{atts = []} = Doc, _RevPos) ->
+ Doc;
+set_recreated_att_revpos(Db, #doc{atts = Atts, meta = Meta} = Doc, RevPos) ->
+ Atts1 = [couch_att:disk_term_set_revpos(Att, RevPos) || Att <- Atts],
+ case lists:keytake(comp_body, 1, Meta) of
+ {value, {comp_body, CompBody}, Meta1} ->
+ % comp_body existence means the doc summary was already serialized
+ % with old atts. We can't use that, we need to re-serialize it with
+ % the new atts. keytake removes the old comp_body and then
+ % serialize_doc will re-create it.
+ Doc1 = Doc#doc{body = CompBody, atts = Atts1, meta = Meta1},
+ couch_db_engine:serialize_doc(Db, Doc1);
+ false ->
+ % This is in the odd case we use a different engine than the b-tree
+ % engine which doesn't need the comp_body hack. Just update the
+ % atts then.
+ Doc#doc{atts = Atts1}
+ end.
+
is_overflowed(_New, _Old, []) ->
false;
is_overflowed(Old, Old, _FullPartitions) ->
@@ -697,7 +726,8 @@ update_docs_int(Db, DocsList, LocalDocs, ReplicatedChanges)
->
add_infos = [],
rem_seqs = [],
cur_seq = UpdateSeq,
- full_partitions = FullPartitions
+ full_partitions = FullPartitions,
+ db = Db
},
{ok, AccOut} = merge_rev_trees(DocsList, OldDocInfos, AccIn),
#merge_acc{
diff --git a/src/couch/test/eunit/couch_db_recreate_att_tests.erl
b/src/couch/test/eunit/couch_db_recreate_att_tests.erl
new file mode 100644
index 000000000..bebc13402
--- /dev/null
+++ b/src/couch/test/eunit/couch_db_recreate_att_tests.erl
@@ -0,0 +1,184 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_db_recreate_att_tests).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-define(DOC_ID, ~"doc").
+-define(ATT_NAME, ~"att").
+-define(ATT_TYPE, ~"application/octet-stream").
+-define(ADATA, ~"aaaaa").
+-define(BDATA, ~"bbbbbb").
+
+setup() ->
+ Src = ?tempdb(),
+ Tgt = ?tempdb(),
+ {ok, SrcDb} = couch_db:create(Src, [?ADMIN_CTX]),
+ {ok, TgtDb} = couch_db:create(Tgt, [?ADMIN_CTX]),
+ ok = couch_db:close(SrcDb),
+ ok = couch_db:close(TgtDb),
+ {Src, Tgt}.
+
+teardown({Src, Tgt}) ->
+ ok = couch_server:delete(Src, [?ADMIN_CTX]),
+ ok = couch_server:delete(Tgt, [?ADMIN_CTX]).
+
+recreate_att_test_() ->
+ {
+ "Recreating doc attachments",
+ {
+ setup,
+ fun() -> test_util:start_couch([ioq]) end,
+ fun test_util:stop_couch/1,
+ {
+ foreach,
+ fun setup/0,
+ fun teardown/1,
+ [
+ ?TDEF_FE(t_recreate_revpos),
+ ?TDEF_FE(t_recreate_multiple_deletes),
+ ?TDEF_FE(t_atts_since_works),
+ ?TDEF_FE(t_recreate_replicates),
+ ?TDEF_FE(t_recreate_no_atts),
+ ?TDEF_FE(t_revs_depend_on_att_content),
+ ?TDEF_FE(t_rev_is_deterministic)
+ ]
+ }
+ }
+ }.
+
+t_recreate_revpos({Src, _}) ->
+ {1, _} = Rev1 = create(Src, ?ADATA),
+ {2, _} = delete(Src, Rev1),
+ {3, _} = Rev3 = create(Src, ?BDATA),
+ Att = open_att(Src, Rev3, []),
+ ?assertEqual(3, couch_att:fetch(revpos, Att)),
+ ?assertEqual(?BDATA, couch_att:to_binary(Att)).
+
+t_recreate_multiple_deletes({Src, _}) ->
+ Rev1 = create(Src, ?ADATA),
+ Rev2 = delete(Src, Rev1),
+ Rev3 = create(Src, ?BDATA),
+ {4, _} = delete(Src, Rev3),
+ {5, _} = Rev5 = create(Src, ?ADATA),
+ ?assertNotEqual(Rev2, Rev5),
+ Att = open_att(Src, Rev5, []),
+ ?assertEqual(5, couch_att:fetch(revpos, Att)),
+ % a's then b's then a's again
+ ?assertEqual(?ADATA, couch_att:to_binary(Att)).
+
+t_atts_since_works({Src, _}) ->
+ Rev1 = create(Src, ?ADATA),
+ delete(Src, Rev1),
+ Rev3 = create(Src, ?BDATA),
+ Att = open_att(Src, Rev3, [{atts_since, [Rev1]}]),
+ ?assertNotEqual(stub, couch_att:fetch(data, Att)),
+ ?assertEqual(?BDATA, couch_att:to_binary(Att)).
+
+t_recreate_replicates({Src, Tgt}) ->
+ % Replicate a's
+ Rev1 = create(Src, ?ADATA),
+ replicate(Src, Tgt, Rev1, []),
+ ?assertEqual(?ADATA, couch_att:to_binary(open_att(Tgt, Rev1, []))),
+ % Delete, recreate and re-replicate
+ delete(Src, Rev1),
+ Rev3 = create(Src, ?BDATA),
+ replicate(Src, Tgt, Rev3, [Rev1]),
+ % Expect b's now
+ ?assertEqual(?BDATA, couch_att:to_binary(open_att(Tgt, Rev3, []))).
+
+t_recreate_no_atts({Src, _}) ->
+ Rev1 = create(Src, ?ADATA),
+ delete(Src, Rev1),
+ {ok, Db} = couch_db:open_int(Src, []),
+ Doc = #doc{id = ?DOC_ID, body = {[{~"foo", 1}]}},
+ {ok, {3, _}} = couch_db:update_doc(Db, Doc, []),
+ ok = couch_db:close(Db),
+ {ok, Db1} = couch_db:open_int(Src, []),
+ {ok, #doc{atts = Atts, body = Body}} = couch_db:open_doc(Db1, ?DOC_ID, []),
+ ok = couch_db:close(Db1),
+ ?assertEqual([], Atts),
+ ?assertEqual({[{~"foo", 1}]}, Body).
+
+t_revs_depend_on_att_content({Db1, Db2}) ->
+ % Recreate with different attachment content in each db. We should be
+ % getting different revisions then, too.
+ do_recreate(Db1, Db2),
+ {3, _} = Rev1 = create(Db1, ?ADATA),
+ {3, _} = Rev2 = create(Db2, ?BDATA),
+ ?assertNotEqual(Rev1, Rev2).
+
+t_rev_is_deterministic({Db1, Db2}) ->
+ % Write some random attachment to get a different file position in db2 and
+ % we should still get the same revision for same content. The idea is we
+ % can generate the data independently on multuple servers but their revs
+ % should match (as opposed to creating conflicts)
+ create(Db2, ~"junk", [{?ATT_NAME, ?BDATA}]),
+ do_recreate(Db1, Db2),
+ {3, _} = Rev = create(Db1, ?BDATA),
+ ?assertEqual(Rev, create(Db2, ?BDATA)).
+
+% The revisions should match, so a recreated doc will branch off the same
+% deleted revision in both dbs.
+do_recreate(Db1, Db2) ->
+ Rev1 = create(Db1, ?ADATA),
+ ?assertEqual(Rev1, create(Db2, ?ADATA)),
+ Rev2 = delete(Db1, Rev1),
+ ?assertEqual(Rev2, delete(Db2, Rev1)).
+
+create(DbName, Data) ->
+ create(DbName, ?DOC_ID, [{?ATT_NAME, Data}]).
+
+create(DbName, DocId, Atts) ->
+ AttsJson = [
+ {Name, {[{~"content_type", ?ATT_TYPE}, {~"data",
base64:encode(Data)}]}}
+ || {Name, Data} <- Atts
+ ],
+ Doc = couch_doc:from_json_obj(
+ {[
+ {~"_id", DocId},
+ {~"_attachments", {AttsJson}}
+ ]}
+ ),
+ {ok, Db} = couch_db:open_int(DbName, []),
+ {ok, Rev} = couch_db:update_doc(Db, Doc, []),
+ ok = couch_db:close(Db),
+ Rev.
+
+delete(DbName, {Pos, RevId}) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ Doc = #doc{id = ?DOC_ID, revs = {Pos, [RevId]}, deleted = true},
+ {ok, Rev} = couch_db:update_doc(Db, Doc, []),
+ ok = couch_db:close(Db),
+ Rev.
+
+open_att(DbName, Rev, Opts) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ {ok, [{ok, #doc{atts = [Att]}}]} = couch_db:open_doc_revs(Db, ?DOC_ID,
[Rev], Opts),
+ ok = couch_db:close(Db),
+ Att.
+
+% Pretend we're replicating. Fetch revs with atts_since that target has. Write
+% to target with new_edits=false. Attachments which the source thinks the
+% target already has are sent as stubs.
+replicate(Src, Tgt, Rev, AttsSince) ->
+ {ok, SDb} = couch_db:open_int(Src, []),
+ Opts = [revs, {atts_since, AttsSince}],
+ {ok, [{ok, Doc}]} = couch_db:open_doc_revs(SDb, ?DOC_ID, [Rev], Opts),
+ ok = couch_db:close(SDb),
+ Json = couch_doc:to_json_obj(Doc, [revs, attachments]),
+ Doc1 = couch_doc:from_json_obj(Json),
+ {ok, TDb} = couch_db:open_int(Tgt, []),
+ {ok, []} = couch_db:update_docs(TDb, [Doc1], [], ?REPLICATED_CHANGES),
+ ok = couch_db:close(TDb).