This is an automated email from the ASF dual-hosted git repository.

nickva pushed a commit to branch split-batch-bytes-vheap
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 322a39847d96e58a318a90013e93babac127f1a5
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Wed Nov 11 18:24:32 2026 -0500

    Adjust a few shard splitting parameters
    
    After seeing an OTP issue [1] related to bin_vheap_size I played with the 
batch
    size and min_bin_vheap_size on the split process and noticed a nice speedup 
if
    we both lowered the batch size and used a higher min_bin_vheap_size size.
    
    [1] https://github.com/erlang/otp/issues/11841
---
 src/couch/src/couch_db_split.erl              | 17 ++++++++++++++++-
 src/couch/test/eunit/couch_db_split_tests.erl | 14 ++++++++++++--
 2 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/src/couch/src/couch_db_split.erl b/src/couch/src/couch_db_split.erl
index 67f31ea1b..f41f88eb6 100644
--- a/src/couch/src/couch_db_split.erl
+++ b/src/couch/src/couch_db_split.erl
@@ -24,7 +24,15 @@
 
 % Source doc batching parameters
 -define(BODY_BATCH_COUNT, 1000).
--define(BODY_BATCH_BYTES, 16777216).
+-define(BODY_BATCH_BYTES, 1048576).
+
+% Min binary virtual heap size of the splitting process. The default of ~370KB
+% is garbage collected a few times for each batch of bodies. Anything still in
+% use at that point will move to the old generation and only a full sweep will
+% free it after that. So we're increasing the minimum here to keep them in the
+% young heap. Benchmarking on OTP 27 showed a good speedup vs the default (63s
+% -> 44sec).
+-define(MIN_BIN_VHEAP_BYTES, 8388608).
 
 -record(state, {
     source_db,
@@ -65,6 +73,7 @@ split(Source, #{} = Targets, PickFun) when
             Engine = get_engine(SourceDb),
             Partitioned = couch_db:is_partitioned(SourceDb),
             HashFun = mem3_hash:get_hash_fun(couch_db:name(SourceDb)),
+            set_min_bin_vheap_size(),
             try
                 split(SourceDb, Partitioned, Engine, Targets, PickFun, HashFun)
             catch
@@ -572,6 +581,12 @@ total_sizes(#racc{active = Active, external = External, 
atts = Atts}) ->
     TotalAtts = lists:foldl(fun({_, S}, A) -> S + A end, 0, Atts),
     {Active + TotalAtts, External + TotalAtts}.
 
+% We have to set it as a process flag as spawn_opt has a bug currently
+% (see https://github.com/erlang/otp/issues/11840)
+set_min_bin_vheap_size() ->
+    Words = ?MIN_BIN_VHEAP_BYTES div erlang:system_info(wordsize),
+    process_flag(min_bin_vheap_size, Words).
+
 get_max_buffer_size() ->
     config:get_integer("reshard", "split_buffer_size", ?DEFAULT_BUFFER_SIZE).
 
diff --git a/src/couch/test/eunit/couch_db_split_tests.erl 
b/src/couch/test/eunit/couch_db_split_tests.erl
index 510a2321f..ff816b612 100644
--- a/src/couch/test/eunit/couch_db_split_tests.erl
+++ b/src/couch/test/eunit/couch_db_split_tests.erl
@@ -112,7 +112,7 @@ should_keep_all_the_docs(DbName) ->
     PickFun = fun(DocId, Rs, _HashFun) -> lists:nth(1 + erlang:phash2(DocId, 
2), Rs) end,
     {timeout, ?TIMEOUT,
         ?_test(begin
-            ?assertEqual(1204, length(Before)),
+            ?assertEqual(1209, length(Before)),
             ?assertEqual(2, length(proplists:get_value(~"conflict", Before))),
             {ok, _} = couch_db_split:split(DbName, TMap, PickFun),
             [SnapA, SnapB] = [all_the_docs(Name) || Name <- maps:values(TMap)],
@@ -411,13 +411,23 @@ bunch_of_docs(DbName, N) ->
         ]}
     ),
     Big = couch_doc:from_json_obj({[{~"_id", ~"big"}, {~"blob", 
binary:copy(~"y", 200000)}]}),
+    % Incompressible, to get over the source batch byte limit
+    Rand = [
+        couch_doc:from_json_obj(
+            {[
+                {~"_id", <<"rand", (integer_to_binary(I))/binary>>},
+                {~"blob", base64:encode(crypto:strong_rand_bytes(300000))}
+            ]}
+        )
+     || I <- lists:seq(1, 5)
+    ],
     Del = couch_doc:from_json_obj({[{~"_id", ~"del"}, {~"v", 1}]}),
     Conflicts = [
         #doc{id = ~"conflict", revs = {1, [~"aaa"]}, body = {[{~"v", ~"a"}]}},
         #doc{id = ~"conflict", revs = {1, [~"bbb"]}, body = {[{~"v", ~"b"}]}}
     ],
     couch_util:with_db(DbName, fun(Db) ->
-        {ok, _} = couch_db:update_docs(Db, Plain ++ [Att, Big]),
+        {ok, _} = couch_db:update_docs(Db, Plain ++ [Att, Big] ++ Rand),
         {ok, Db1} = couch_db:reopen(Db),
         {ok, [{ok, {Pos, RevId}}]} = couch_db:update_docs(Db1, [Del]),
         {ok, Db2} = couch_db:reopen(Db1),

Reply via email to