This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new 0a666d0fe Adjust a few shard splitting parameters
0a666d0fe is described below
commit 0a666d0feb636eeff4c0cf728d55d75e882658e1
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Wed Nov 11 18:24:32 2026 -0500
Adjust a few shard splitting parameters
After seeing an OTP issue [1] related to bin_vheap_size I played with the
batch
size and min_bin_vheap_size on the split process and noticed a nice speedup
if
we both lowered the batch size and used a higher min_bin_vheap_size size.
[1] https://github.com/erlang/otp/issues/11841
---
src/couch/src/couch_db_split.erl | 17 ++++++++++++++++-
src/couch/test/eunit/couch_db_split_tests.erl | 14 ++++++++++++--
2 files changed, 28 insertions(+), 3 deletions(-)
diff --git a/src/couch/src/couch_db_split.erl b/src/couch/src/couch_db_split.erl
index 67f31ea1b..f41f88eb6 100644
--- a/src/couch/src/couch_db_split.erl
+++ b/src/couch/src/couch_db_split.erl
@@ -24,7 +24,15 @@
% Source doc batching parameters
-define(BODY_BATCH_COUNT, 1000).
--define(BODY_BATCH_BYTES, 16777216).
+-define(BODY_BATCH_BYTES, 1048576).
+
+% Min binary virtual heap size of the splitting process. The default of ~370KB
+% is garbage collected a few times for each batch of bodies. Anything still in
+% use at that point will move to the old generation and only a full sweep will
+% free it after that. So we're increasing the minimum here to keep them in the
+% young heap. Benchmarking on OTP 27 showed a good speedup vs the default (63s
+% -> 44sec).
+-define(MIN_BIN_VHEAP_BYTES, 8388608).
-record(state, {
source_db,
@@ -65,6 +73,7 @@ split(Source, #{} = Targets, PickFun) when
Engine = get_engine(SourceDb),
Partitioned = couch_db:is_partitioned(SourceDb),
HashFun = mem3_hash:get_hash_fun(couch_db:name(SourceDb)),
+ set_min_bin_vheap_size(),
try
split(SourceDb, Partitioned, Engine, Targets, PickFun, HashFun)
catch
@@ -572,6 +581,12 @@ total_sizes(#racc{active = Active, external = External,
atts = Atts}) ->
TotalAtts = lists:foldl(fun({_, S}, A) -> S + A end, 0, Atts),
{Active + TotalAtts, External + TotalAtts}.
+% We have to set it as a process flag as spawn_opt has a bug currently
+% (see https://github.com/erlang/otp/issues/11840)
+set_min_bin_vheap_size() ->
+ Words = ?MIN_BIN_VHEAP_BYTES div erlang:system_info(wordsize),
+ process_flag(min_bin_vheap_size, Words).
+
get_max_buffer_size() ->
config:get_integer("reshard", "split_buffer_size", ?DEFAULT_BUFFER_SIZE).
diff --git a/src/couch/test/eunit/couch_db_split_tests.erl
b/src/couch/test/eunit/couch_db_split_tests.erl
index 510a2321f..ff816b612 100644
--- a/src/couch/test/eunit/couch_db_split_tests.erl
+++ b/src/couch/test/eunit/couch_db_split_tests.erl
@@ -112,7 +112,7 @@ should_keep_all_the_docs(DbName) ->
PickFun = fun(DocId, Rs, _HashFun) -> lists:nth(1 + erlang:phash2(DocId,
2), Rs) end,
{timeout, ?TIMEOUT,
?_test(begin
- ?assertEqual(1204, length(Before)),
+ ?assertEqual(1209, length(Before)),
?assertEqual(2, length(proplists:get_value(~"conflict", Before))),
{ok, _} = couch_db_split:split(DbName, TMap, PickFun),
[SnapA, SnapB] = [all_the_docs(Name) || Name <- maps:values(TMap)],
@@ -411,13 +411,23 @@ bunch_of_docs(DbName, N) ->
]}
),
Big = couch_doc:from_json_obj({[{~"_id", ~"big"}, {~"blob",
binary:copy(~"y", 200000)}]}),
+ % Incompressible, to get over the source batch byte limit
+ Rand = [
+ couch_doc:from_json_obj(
+ {[
+ {~"_id", <<"rand", (integer_to_binary(I))/binary>>},
+ {~"blob", base64:encode(crypto:strong_rand_bytes(300000))}
+ ]}
+ )
+ || I <- lists:seq(1, 5)
+ ],
Del = couch_doc:from_json_obj({[{~"_id", ~"del"}, {~"v", 1}]}),
Conflicts = [
#doc{id = ~"conflict", revs = {1, [~"aaa"]}, body = {[{~"v", ~"a"}]}},
#doc{id = ~"conflict", revs = {1, [~"bbb"]}, body = {[{~"v", ~"b"}]}}
],
couch_util:with_db(DbName, fun(Db) ->
- {ok, _} = couch_db:update_docs(Db, Plain ++ [Att, Big]),
+ {ok, _} = couch_db:update_docs(Db, Plain ++ [Att, Big] ++ Rand),
{ok, Db1} = couch_db:reopen(Db),
{ok, [{ok, {Pos, RevId}}]} = couch_db:update_docs(Db1, [Del]),
{ok, Db2} = couch_db:reopen(Db1),