This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new ffe27c171 Batch up writes for shard splitting
ffe27c171 is described below
commit ffe27c171fce708c809715d8a6388a15566fb6d8
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Thu Nov 12 02:15:58 2026 -0500
Batch up writes for shard splitting
Similar to what we did for reads (d61d3e0c4) just for writes. Export
append_raw_chunks/2 couch_file call and add write_doc_bodies/2` to the couch
engine API.
Trying it with Q=1 shards 1GB file with about 1KB docs:
main: 58 sec
pr: 45 sec
---
src/couch/src/couch_bt_engine.erl | 7 ++
src/couch/src/couch_db_engine.erl | 13 +++
src/couch/src/couch_db_split.erl | 93 +++++++++++++++-------
src/couch/src/couch_file.erl | 2 +-
.../src/cpse_test_read_write_docs.erl | 38 +++++++++
5 files changed, 122 insertions(+), 31 deletions(-)
diff --git a/src/couch/src/couch_bt_engine.erl
b/src/couch/src/couch_bt_engine.erl
index da7e34bb4..9c15a32a0 100644
--- a/src/couch/src/couch_bt_engine.erl
+++ b/src/couch/src/couch_bt_engine.erl
@@ -61,6 +61,7 @@
serialize_doc/2,
write_doc_body/2,
+ write_doc_bodies/2,
write_doc_infos/3,
purge_docs/3,
copy_purge_infos/2,
@@ -452,6 +453,12 @@ write_doc_body(St, #doc{} = Doc) ->
{ok, Ptr, Written} = couch_file:append_raw_chunk(Fd, Doc#doc.body),
{ok, Doc#doc{body = Ptr}, Written}.
+write_doc_bodies(#st{fd = Fd}, Docs) when is_list(Docs) ->
+ Chunks = [Chunk || #doc{body = Chunk} <- Docs],
+ {ok, Resps} = couch_file:append_raw_chunks(Fd, Chunks),
+ Fun = fun(Doc, {Ptr, Written}) -> {Doc#doc{body = Ptr}, Written} end,
+ {ok, lists:zipwith(Fun, Docs, Resps)}.
+
write_doc_infos(#st{} = St, Pairs, LocalDocs) ->
#st{
id_tree = IdTree,
diff --git a/src/couch/src/couch_db_engine.erl
b/src/couch/src/couch_db_engine.erl
index 91f5a9859..f175e63b7 100644
--- a/src/couch/src/couch_db_engine.erl
+++ b/src/couch/src/couch_db_engine.erl
@@ -384,6 +384,14 @@
-callback write_doc_body(DbHandle :: db_handle(), Doc :: doc()) ->
{ok, FlushedDoc :: doc(), BytesWritten :: non_neg_integer()}.
+% Batch version of write_doc_body/2. It should be called with a list of
+% serialized docs. It should return a {FlushedDoc, BytesWritten} pair per input
+% document, in order. Engines implementations maybe write all the bodies
+% together (couch_bt_engine appends them with a one couch_file call, for
+% example).
+-callback write_doc_bodies(DbHandle :: db_handle(), Docs :: [doc()]) ->
+ {ok, [{FlushedDoc :: doc(), BytesWritten :: non_neg_integer()}]}.
+
% This function is called from the context of couch_db_updater
% and as such is guaranteed single threaded for the given
% DbHandle.
@@ -717,6 +725,7 @@
serialize_doc/2,
write_doc_body/2,
+ write_doc_bodies/2,
write_doc_infos/3,
purge_docs/3,
copy_purge_infos/2,
@@ -928,6 +937,10 @@ write_doc_body(#db{} = Db, #doc{} = Doc) ->
#db{engine = {Engine, EngineState}} = Db,
Engine:write_doc_body(EngineState, Doc).
+write_doc_bodies(#db{} = Db, Docs) when is_list(Docs) ->
+ #db{engine = {Engine, EngineState}} = Db,
+ Engine:write_doc_bodies(EngineState, Docs).
+
write_doc_infos(#db{} = Db, DocUpdates, LocalDocs) ->
#db{engine = {Engine, EngineState}} = Db,
{ok, NewSt} = Engine:write_doc_infos(EngineState, DocUpdates, LocalDocs),
diff --git a/src/couch/src/couch_db_split.erl b/src/couch/src/couch_db_split.erl
index f41f88eb6..184e71b8b 100644
--- a/src/couch/src/couch_db_split.erl
+++ b/src/couch/src/couch_db_split.erl
@@ -58,6 +58,7 @@
source_db,
target_db,
bodies = #{},
+ docs = [],
active = 0,
external = 0,
atts = []
@@ -426,25 +427,32 @@ changes_cb(#full_doc_info{} = FDI, #state{} = State) ->
% Use batched make_docs read to get all the pending docs bodies
copy_pending(#state{pending = []} = State) ->
State;
-copy_pending(#state{pending = Pending, source_db = SourceDb} = State) ->
+copy_pending(#state{pending = Pending, source_db = SourceDb, targets =
Targets} = State) ->
FDIs = lists:reverse(Pending),
Bodies = get_bodies(SourceDb, FDIs),
- State1 = State#state{pending = [], pending_count = 0, pending_bytes = 0},
- lists:foldl(
- fun(FDI, StateAcc) ->
- copy_fdi(FDI, Bodies, StateAcc)
+ KeyFun = fun(#full_doc_info{id = Id}) ->
+ {Key, #target{}} = pick_target(Id, State, Targets, changes),
+ Key
+ end,
+ Targets1 = maps:fold(
+ fun(Key, KeyFDIs, Acc) ->
+ Acc#{Key := copy_fdis(KeyFDIs, Bodies, map_get(Key, Acc), State)}
end,
- State1,
- FDIs
- ).
-
-copy_fdi(#full_doc_info{id = Id} = FDI, #{} = Bodies, #state{} = State) ->
- #state{source_db = SourceDb, targets = Targets} = State,
- {Key, Target} = pick_target(Id, State, Targets, changes),
- FDI1 = process_fdi(FDI, Bodies, SourceDb, Target#target.db),
- MaxBuffer = State#state.max_buffer_size,
- Target1 = acc_and_flush(FDI1, Target, MaxBuffer, fun commit_docs/1),
- State#state{targets = Targets#{Key => Target1}}.
+ Targets,
+ maps:groups_from_list(KeyFun, FDIs)
+ ),
+ State#state{targets = Targets1, pending = [], pending_count = 0,
pending_bytes = 0}.
+
+% Write the docs bodies as a batch. First serialize the bodies of all the
+% leafs, then batch write them.
+copy_fdis(FDIs, #{} = Bodies, #target{db = TargetDb} = Target, #state{} =
State) ->
+ #state{source_db = SourceDb, max_buffer_size = MaxBuffer} = State,
+ MapF = fun(FDI, Acc) -> serialize_fdi(FDI, Bodies, SourceDb, TargetDb,
Acc) end,
+ {FDIs1, DocsRev} = lists:mapfoldl(MapF, [], FDIs),
+ {ok, Written} = couch_db_engine:write_doc_bodies(TargetDb,
lists:reverse(DocsRev)),
+ {FDIs2, []} = lists:mapfoldl(fun finish_fdi/2, Written, FDIs1),
+ FoldF = fun(FDI, Acc) -> acc_and_flush(FDI, Acc, MaxBuffer, fun
commit_docs/1) end,
+ lists:foldl(FoldF, Target, FDIs2).
% Get a map of Ptr => #doc{} from a list if FDIs
get_bodies(SourceDb, FDIs) ->
@@ -476,19 +484,23 @@ commit_docs(#target{buffer = FDIs, db = Db} = Target) ->
{ok, Db2} = couch_db_engine:commit_data(Db1),
Target#target{buffer = [], buffer_size = 0, db = Db2}.
-process_fdi(FDI, #{} = Bodies, SourceDb, TargetDb) ->
+% Copy attachments and serialize bodies. Leafs get their external and
+% attachment sizes here, then keep the source pointer until finish_fdi/2 call.
+serialize_fdi(FDI, #{} = Bodies, SourceDb, TargetDb, DocsAcc) ->
#full_doc_info{id = Id, rev_tree = RTree} = FDI,
- Acc = #racc{id = Id, source_db = SourceDb, target_db = TargetDb, bodies =
Bodies},
- {NewRTree, NewAcc} = couch_key_tree:mapfold(fun revtree_cb/4, Acc, RTree),
- {Active, External} = total_sizes(NewAcc),
- FDI#full_doc_info{
- rev_tree = NewRTree,
- sizes = #size_info{active = Active, external = External}
- }.
+ Acc = #racc{
+ id = Id,
+ source_db = SourceDb,
+ target_db = TargetDb,
+ bodies = Bodies,
+ docs = DocsAcc
+ },
+ {NewRTree, #racc{docs = DocsAcc1}} = couch_key_tree:mapfold(fun
serialize_cb/4, Acc, RTree),
+ {FDI#full_doc_info{rev_tree = NewRTree}, DocsAcc1}.
-revtree_cb(_Rev, _Leaf, branch, Acc) ->
+serialize_cb(_Rev, _Leaf, branch, Acc) ->
{[], Acc};
-revtree_cb({Pos, RevId}, Leaf, leaf, Acc) ->
+serialize_cb({Pos, RevId}, Leaf, leaf, Acc) ->
#racc{id = Id, source_db = SourceDb, target_db = TargetDb, bodies =
Bodies} = Acc,
#leaf{deleted = Deleted, ptr = Ptr, sizes = LeafSizes} = Leaf,
#doc{body = Body, atts = AttInfos0} = maps:get(Ptr, Bodies),
@@ -516,16 +528,37 @@ revtree_cb({Pos, RevId}, Leaf, leaf, Acc) ->
Atts = [process_attachment(Att, SourceDb, TargetDb) || Att <- AttInfos],
Doc2 = Doc1#doc{atts = Atts},
Doc3 = couch_db_engine:serialize_doc(TargetDb, Doc2),
- {ok, Doc4, Active} = couch_db_engine:write_doc_body(TargetDb, Doc3),
% element(3,...) and (4,...) are the stream pointer and size respecitively
% (see couch_att.erl) They are numeric for compatibility with older formats
AttSizes = [{element(3, A), element(4, A)} || A <- Atts],
NewLeaf = Leaf#leaf{
- ptr = Doc4#doc.body,
- sizes = #size_info{active = Active, external = External},
+ sizes = #size_info{external = External},
atts = AttSizes
},
- {NewLeaf, add_sizes(Active, External, AttSizes, Acc)}.
+ {NewLeaf, Acc#racc{docs = [Doc3 | Acc#racc.docs]}}.
+
+% Second pass over the leafs. Bodies are written already. Update sizes and
+% turn #doc{} records into proper leafs with ptr and size data.
+finish_fdi(#full_doc_info{rev_tree = RTree} = FDI, Written) ->
+ {NewRTree, {Written1, Acc}} = couch_key_tree:mapfold(
+ fun finish_cb/4, {Written, #racc{}}, RTree
+ ),
+ {Active, External} = total_sizes(Acc),
+ NewFDI = FDI#full_doc_info{
+ rev_tree = NewRTree,
+ sizes = #size_info{active = Active, external = External}
+ },
+ {NewFDI, Written1}.
+
+finish_cb(_Rev, Value, branch, Acc) ->
+ {Value, Acc};
+finish_cb(_Rev, #leaf{} = Leaf, leaf, {[{#doc{body = Ptr}, Active} | Rest],
#racc{} = Acc}) ->
+ #leaf{sizes = #size_info{external = External} = Sizes, atts = AttSizes} =
Leaf,
+ NewLeaf = Leaf#leaf{
+ ptr = Ptr,
+ sizes = Sizes#size_info{active = Active}
+ },
+ {NewLeaf, {Rest, add_sizes(Active, External, AttSizes, Acc)}}.
% This is copied almost verbatim from the compactor
process_attachment(
diff --git a/src/couch/src/couch_file.erl b/src/couch/src/couch_file.erl
index 1a69fa017..112d8ba6a 100644
--- a/src/couch/src/couch_file.erl
+++ b/src/couch/src/couch_file.erl
@@ -43,7 +43,7 @@
-export([open/1, open/2, close/1, bytes/1, sync/1, truncate/2, set_db_pid/2]).
-export([pread_term/2, pread_binary/2]).
-export([append_binary/2]).
--export([append_raw_chunk/2, assemble_file_chunk_and_checksum/1]).
+-export([append_raw_chunk/2, append_raw_chunks/2,
assemble_file_chunk_and_checksum/1]).
-export([append_term/2, append_term/3]).
-export([pread_terms/2]).
-export([append_terms/2, append_terms/3]).
diff --git a/src/couch_pse_tests/src/cpse_test_read_write_docs.erl
b/src/couch_pse_tests/src/cpse_test_read_write_docs.erl
index 16f4283a9..8b1084027 100644
--- a/src/couch_pse_tests/src/cpse_test_read_write_docs.erl
+++ b/src/couch_pse_tests/src/cpse_test_read_write_docs.erl
@@ -304,3 +304,41 @@ cpse_delete_local_doc(Db1) ->
[not_found],
couch_db_engine:open_local_docs(Db3, [<<"_local/foo">>])
).
+
+cpse_write_doc_bodies(Db1) ->
+ {ok, DbBatch} = cpse_util:create_db(),
+ try
+ Docs = [
+ #doc{id = ~"1", body = {[{~"a", 1}]}},
+ #doc{id = ~"2", body = {[{~"b", binary:copy(~"x", 10000)}]}},
+ #doc{id = ~"3", body = {[{~"c", 3}]}}
+ ],
+ SerializeF = fun(Doc) -> couch_db_engine:serialize_doc(Db1, Doc) end,
+ SDocs = lists:map(SerializeF, Docs),
+ IndividualF = fun(Doc) ->
+ {ok, Doc1, Written} = couch_db_engine:write_doc_body(Db1, Doc),
+ {Doc1, Written}
+ end,
+ Individual = lists:map(IndividualF, SDocs),
+ {ok, Batch} = couch_db_engine:write_doc_bodies(DbBatch, SDocs),
+ ?assertEqual(3, length(Batch)),
+ ?assertEqual(
+ [Written || {_, Written} <- Individual],
+ [Written || {_, Written} <- Batch]
+ ),
+ ?assertEqual(
+ [Doc#doc.body || {Doc, _} <- Individual],
+ [Doc#doc.body || {Doc, _} <- Batch]
+ ),
+ ReadBodyF = fun({Doc, _}) ->
+ Doc1 = couch_db_engine:read_doc_body(DbBatch, Doc),
+ couch_compress:decompress(Doc1#doc.body)
+ end,
+ Expected = [Doc#doc.body || Doc <- Docs],
+ Written = lists:map(ReadBodyF, Batch),
+ ?assertEqual(Expected, Written),
+ % Check the empty case
+ ?assertEqual({ok, []}, couch_db_engine:write_doc_bodies(DbBatch, []))
+ after
+ ok = couch_server:delete(couch_db:name(DbBatch), [])
+ end.