This is an automated email from the ASF dual-hosted git repository.
nickva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new 2b1175841 Unify membership hashes
2b1175841 is described below
commit 2b117584198aa60b61822125742b2ba68883daef
Author: Nick Vatamaniuc <[email protected]>
AuthorDate: Thu Jun 11 19:27:04 2026 -0400
Unify membership hashes
Previously replicator and fabric_doc_updater used different membership
algorithms to consistently pick a first node order from a range of shard
copies.
Unifying them simplifies the code and also fixes issues like #6029.
While at it move away from crc32 and used a new phash2/1 function. We don't
have to create a term_to_binary for it first as it can take regular Erlang
terms.
(Thanks to Robert Newson for the original idea to do this)
Fix #6029
---
src/couch_peruser/src/couch_peruser.erl | 5 +--
.../src/couch_replicator_utils.erl | 6 +---
src/fabric/src/fabric_doc_update.erl | 18 +++++++++-
src/mem3/src/mem3.erl | 39 ++++++++++++++++++----
src/mem3/src/mem3_util.erl | 4 +--
5 files changed, 53 insertions(+), 19 deletions(-)
diff --git a/src/couch_peruser/src/couch_peruser.erl
b/src/couch_peruser/src/couch_peruser.erl
index 8a7cbe13a..bb868d9f7 100644
--- a/src/couch_peruser/src/couch_peruser.erl
+++ b/src/couch_peruser/src/couch_peruser.erl
@@ -269,10 +269,7 @@ should_handle_doc(ShardName, DocId) ->
) -> boolean().
should_handle_doc_int(ShardName, DocId) ->
DbName = mem3:dbname(ShardName),
- Live = [erlang:node() | nodes()],
- Shards = mem3:shards(DbName, DocId),
- Nodes = [N || #shard{node = N} <- Shards, lists:member(N, Live)],
- case mem3:owner(DbName, DocId, Nodes) of
+ case mem3:owner(DbName, DocId) of
ThisNode when ThisNode =:= node() ->
couch_log:debug("peruser: handling ~s/~s", [DbName, DocId]),
% do the database action
diff --git a/src/couch_replicator/src/couch_replicator_utils.erl
b/src/couch_replicator/src/couch_replicator_utils.erl
index d784b8a84..1ce5b1de0 100644
--- a/src/couch_replicator/src/couch_replicator_utils.erl
+++ b/src/couch_replicator/src/couch_replicator_utils.erl
@@ -52,11 +52,7 @@
-spec owner(Dbname :: binary(), DocId :: binary()) -> node().
owner(<<"shards/", _/binary>> = ShardName, DocId) ->
- DbName = mem3:dbname(ShardName),
- Live = [node() | nodes()],
- Shards = mem3:shards(DbName, DocId),
- Nodes = [N || #shard{node = N} <- Shards, lists:member(N, Live)],
- mem3:owner(DbName, DocId, Nodes);
+ mem3:owner(mem3:dbname(ShardName), DocId);
owner(_ShardName, _DocId) ->
node().
diff --git a/src/fabric/src/fabric_doc_update.erl
b/src/fabric/src/fabric_doc_update.erl
index 96f7c1b11..f0481b1fc 100644
--- a/src/fabric/src/fabric_doc_update.erl
+++ b/src/fabric/src/fabric_doc_update.erl
@@ -352,7 +352,7 @@ group_docs_by_shard(DbName, Docs) ->
[] ->
[];
[#shard{range = Range} | _] = Shards ->
- mem3_util:rotate_list({Range, DbName}, Shards)
+ owner_order(DbName, Range, Shards)
end
end
)
@@ -362,6 +362,10 @@ group_docs_by_shard(DbName, Docs) ->
)
).
+owner_order(DbName, Range, Shards) ->
+ Owners = mem3:owners(DbName, Range, [N || #shard{node = N} <- Shards]),
+ [S || N <- Owners, #shard{node = N1} = S <- Shards, N1 =:= N].
+
append_update_replies([], [], DocReplyDict) ->
DocReplyDict;
append_update_replies([Doc | Rest], [], Dict0) ->
@@ -1144,6 +1148,18 @@ sws_false_mode_ok_can_outvote_conflict() ->
lists:sort(Reply)
).
+owner_order_test() ->
+ S1 = #shard{name = <<"r1">>, node = n1, range = [0, 10]},
+ S2 = #shard{name = <<"r1">>, node = n2, range = [0, 10]},
+ S3 = #shard{name = <<"r1">>, node = n3, range = [0, 10]},
+ % The rotation amounts, erlang:phash2({Db, Range}) rem 3, are 0 for
+ % {<<"dba">>, [0, 10]} and 2 for {<<"dbe">>, [0, 10]}
+ ?assertEqual([S1, S2, S3], owner_order(<<"dba">>, [0, 10], [S2, S3, S1])),
+ ?assertEqual([S3, S1, S2], owner_order(<<"dbe">>, [0, 10], [S2, S3, S1])),
+ % the first copy is on the owner node
+ ?assertEqual(n1, hd(mem3:owners(<<"dba">>, [0, 10], [n1, n2, n3]))),
+ ?assertEqual(n3, hd(mem3:owners(<<"dbe">>, [0, 10], [n1, n2, n3]))).
+
% needed for testing to avoid having to start the mem3 application
group_docs_by_shard_hack(_DbName, Shards, Docs) ->
dict:to_list(
diff --git a/src/mem3/src/mem3.erl b/src/mem3/src/mem3.erl
index 6a96ae2a9..b16240665 100644
--- a/src/mem3/src/mem3.erl
+++ b/src/mem3/src/mem3.erl
@@ -31,7 +31,7 @@
-export([compare_nodelists/0, compare_shards/1]).
-export([quorum/1, group_by_proximity/1]).
-export([live_shards/2]).
--export([belongs/2, owner/3]).
+-export([belongs/2, owner/2, owners/3]).
-export([get_placement/1]).
-export([ping/1, ping/2]).
-export([ping_nodes/0, ping_nodes/1, ping_nodes/2]).
@@ -414,11 +414,26 @@ name(#shard{name = Name}) ->
name(#ordered_shard{name = Name}) ->
Name.
-% Direct calculation of node membership. This is the algorithm part. It
-% doesn't read the shard map, just picks owner based on a hash.
--spec owner(binary(), binary(), [node()]) -> node().
-owner(DbName, DocId, Nodes) ->
- hd(mem3_util:rotate_list({DbName, DocId}, lists:usort(Nodes))).
+% Owner node order rotated by {DbName, Range}.
+-spec owners(binary(), binary()) -> [node()].
+owners(DbName, DocId) ->
+ [#shard{range = Range} | _] = Shards = shards(DbName, DocId),
+ owners(DbName, Range, [N || #shard{node = N} <- Shards]).
+
+% Ownership calculation. This is the algorithm part. It doesn't read the shard
+% map, just rotates the given nodes based on a hash of {DbName, Range}.
+-spec owners(binary(), [non_neg_integer()], [node()]) -> [node()].
+owners(DbName, Range, Nodes) ->
+ mem3_util:rotate_list({DbName, Range}, lists:usort(Nodes)).
+
+% Pick the owner node as first live node in the owners/2 order. When a node
+% goes down, ownership of its docs funnels to the next live node in the list.
+-spec owner(binary(), binary()) -> node().
+owner(DbName, DocId) ->
+ live_owner(owners(DbName, DocId), [node() | erlang:nodes()]).
+
+live_owner(Owners, Live) ->
+ hd([N || N <- Owners, lists:member(N, Live)]).
%% Check whether a node is up or down
%% side effect: set up a connection to Node if there not yet is one.
@@ -610,6 +625,18 @@ allowed_nodes_test_() ->
}
]}.
+live_owner_test() ->
+ Owners = [n2, n3, n1],
+ % all nodes live: the head of the try-order owns the doc
+ ?assertEqual(n2, live_owner(Owners, [n1, n2, n3])),
+ % owner down: its docs funnel to the next node in the try-order
+ ?assertEqual(n3, live_owner(Owners, [n1, n3])),
+ ?assertEqual(n1, live_owner(Owners, [n1])),
+ % other nodes down: ownership of n2's docs is unaffected
+ ?assertEqual(n2, live_owner(Owners, [n2])),
+ % no live copy nodes
+ ?assertError(badarg, live_owner(Owners, [])).
+
rotate_rand_degenerate_test() ->
?assertEqual([1], rotate_rand([1])).
diff --git a/src/mem3/src/mem3_util.erl b/src/mem3/src/mem3_util.erl
index cc728e0ee..7284f6e56 100644
--- a/src/mem3/src/mem3_util.erl
+++ b/src/mem3/src/mem3_util.erl
@@ -324,10 +324,8 @@ is_deleted(Change) ->
rotate_list(_Key, []) ->
[];
-rotate_list(Key, List) when not is_binary(Key) ->
- rotate_list(?term_to_bin(Key), List);
rotate_list(Key, List) ->
- {H, T} = lists:split(erlang:crc32(Key) rem length(List), List),
+ {H, T} = lists:split(erlang:phash2(Key) rem length(List), List),
T ++ H.
downcast(#shard{} = S) ->
