Merge branch '1.3.x'
Conflicts:
src/mem3.erl
src/mem3_cache.erl
src/mem3_nodes.erl
src/mem3_rep.erl
src/mem3_sup.erl
src/mem3_sync.erl
src/mem3_sync_event.erl
BugzID: 13780
Project: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/commit/d39a7915
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/tree/d39a7915
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/diff/d39a7915
Branch: refs/heads/import
Commit: d39a79156950a2993ac988da84c59f79caadf572
Parents: 87cbec9 4704323
Author: Adam Kocoloski <[email protected]>
Authored: Fri Jun 1 12:03:55 2012 -0400
Committer: Adam Kocoloski <[email protected]>
Committed: Fri Jun 1 12:03:55 2012 -0400
----------------------------------------------------------------------
README.md | 8 +-
src/mem3.app.src | 3 +-
src/mem3.erl | 39 ++---
src/mem3_cache.erl | 118 ---------------
src/mem3_nodes.erl | 69 +++++----
src/mem3_rep.erl | 134 ++++++++++++-----
src/mem3_shards.erl | 332 +++++++++++++++++++++++++++++++++++++++++++
src/mem3_sup.erl | 3 +-
src/mem3_sync.erl | 64 ++++++---
src/mem3_sync_event.erl | 4 +-
src/mem3_sync_nodes.erl | 116 +++++++++++++++
src/mem3_util.erl | 23 +--
12 files changed, 661 insertions(+), 252 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/d39a7915/src/mem3.app.src
----------------------------------------------------------------------
diff --cc src/mem3.app.src
index 8844778,6ff46f1..bdc2d49
--- a/src/mem3.app.src
+++ b/src/mem3.app.src
@@@ -3,11 -3,12 +3,12 @@@
{vsn, git},
{mod, {mem3_app, []}},
{registered, [
- mem3_cache,
mem3_events,
mem3_nodes,
+ mem3_shards,
mem3_sync,
+ mem3_sync_nodes,
mem3_sup
]},
- {applications, [kernel, stdlib, sasl, crypto, mochiweb, couch]}
+ {applications, [kernel, stdlib, sasl, crypto, mochiweb, couch, twig]}
]}.
http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/d39a7915/src/mem3.erl
----------------------------------------------------------------------
diff --cc src/mem3.erl
index 77e6372,f7934e1..aad85e4
--- a/src/mem3.erl
+++ b/src/mem3.erl
@@@ -15,13 -15,13 +15,14 @@@
-module(mem3).
-export([start/0, stop/0, restart/0, nodes/0, node_info/2, shards/1, shards/2,
- choose_shards/2, n/1, dbname/1, ushards/1]).
+ choose_shards/2, n/1, dbname/1, ushards/1, ushards/2]).
+ -export([get_shard/3, local_shards/1, fold_shards/2]).
-export([sync_security/0, sync_security/1]).
-export([compare_nodelists/0, compare_shards/1]).
--export([group_by_proximity/1]).
+-export([quorum/1, group_by_proximity/1]).
-include("mem3.hrl").
+-include_lib("couch/include/couch_db.hrl").
start() ->
application:start(mem3).
@@@ -104,38 -97,39 +98,31 @@@ shards(DbName, DocId) when is_list(DbNa
shards(DbName, DocId) when is_list(DocId) ->
shards(DbName, list_to_binary(DocId));
shards(DbName, DocId) ->
- HashKey = mem3_util:hash(DocId),
- Head = #shard{
- name = '_',
- node = '_',
- dbname = DbName,
- range = ['$1','$2'],
- ref = '_'
- },
- Conditions = [{'=<', '$1', HashKey}, {'=<', HashKey, '$2'}],
- try ets:select(partitions, [{Head, Conditions, ['$_']}]) of
- [] ->
- mem3_util:load_shards_from_disk(DbName, DocId);
- Shards ->
- Shards
- catch error:badarg ->
- mem3_util:load_shards_from_disk(DbName, DocId)
- end.
+ mem3_shards:for_docid(DbName, DocId).
+-spec ushards(DbName::iodata()) -> [#shard{}].
ushards(DbName) ->
- {L,S,D} = group_by_proximity(live_shards(DbName)),
- % Prefer shards in the local zone over shards in a different zone,
- % but sort each group separately to ensure a consistent choice between
- % nodes in the same zone.
- Shards = lists:sort(L ++ S) ++ lists:sort(D),
- lists:usort(fun(#shard{name=A}, #shard{name=B}) ->
- A =< B
- end, Shards).
-
-live_shards(DbName) ->
Nodes = [node()|erlang:nodes()],
- [S || #shard{node=Node} = S <- shards(DbName), lists:member(Node, Nodes)].
+ ZoneMap = zone_map(Nodes),
+ ushards(live_shards(DbName, Nodes), ZoneMap).
-group_by_proximity(Shards) ->
- {Local, Remote} = lists:partition(fun(S) -> S#shard.node =:= node() end,
- Shards),
- LocalZone = mem3:node_info(node(), <<"zone">>),
- Fun = fun(S) -> mem3:node_info(S#shard.node, <<"zone">>) =:= LocalZone
end,
- {SameZone, DifferentZone} = lists:partition(Fun, Remote),
- {Local, SameZone, DifferentZone}.
+ushards(Shards0, ZoneMap) ->
+ {L,S,D} = group_by_proximity(Shards0, ZoneMap),
+ % Prefer shards in the local zone over shards in a different zone,
+ % but sort each zone separately to ensure a consistent choice between
+ % nodes in the same zone.
+ Shards = choose_ushards(L ++ S) ++ choose_ushards(D),
+ lists:ukeysort(#shard.range, Shards).
+ get_shard(DbName, Node, Range) ->
+ mem3_shards:get(DbName, Node, Range).
+
+ local_shards(DbName) ->
+ mem3_shards:local(DbName).
+
+ fold_shards(Fun, Acc) ->
+ mem3_shards:fold(Fun, Acc).
+
sync_security() ->
mem3_sync_security:go().
@@@ -208,41 -202,3 +195,41 @@@ apportion(Shares, Acc, Remaining) -
N = Remaining rem length(Acc),
[H|T] = lists:nthtail(N, Acc),
apportion(Shares, lists:sublist(Acc, N) ++ [H+1|T], Remaining - 1).
+
+live_shards(DbName, Nodes) ->
+ [S || #shard{node=Node} = S <- shards(DbName), lists:member(Node, Nodes)].
+
+zone_map(Nodes) ->
+ [{Node, node_info(Node, <<"zone">>)} || Node <- Nodes].
+
+group_by_proximity(Shards) ->
+ Nodes = [N || #shard{node=N} <- lists:ukeysort(#shard.node, Shards)],
+ group_by_proximity(Shards, zone_map(Nodes)).
+
+group_by_proximity(Shards, ZoneMap) ->
+ {Local, Remote} = lists:partition(fun(S) -> S#shard.node =:= node() end,
+ Shards),
+ LocalZone = proplists:get_value(node(), ZoneMap),
+ Fun = fun(S) -> proplists:get_value(S#shard.node, ZoneMap) =:= LocalZone
end,
+ {SameZone, DifferentZone} = lists:partition(Fun, Remote),
+ {Local, SameZone, DifferentZone}.
+
+choose_ushards(Shards) ->
+ Groups = group_by_range(lists:sort(Shards)),
+ Fun = fun(Group, {N, Acc}) ->
+ {N+1, [lists:nth(1 + N rem length(Group), Group) | Acc]} end,
+ {_, Result} = lists:foldl(Fun, {0, []}, Groups),
+ Result.
+
+group_by_range(Shards) ->
+ Groups0 = lists:foldl(fun(#shard{range=Range}=Shard, Dict) ->
+ orddict:append(Range, Shard, Dict) end, orddict:new(), Shards),
+ {_, Groups} = lists:unzip(Groups0),
+ Groups.
+
+% quorum functions
+
+quorum(#db{name=DbName}) ->
+ quorum(DbName);
+quorum(DbName) ->
- n(DbName) div 2 + 1.
++ n(DbName) div 2 + 1.
http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/d39a7915/src/mem3_nodes.erl
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/d39a7915/src/mem3_sup.erl
----------------------------------------------------------------------
diff --cc src/mem3_sup.erl
index 07b9498,ae38d7d..bad651a
--- a/src/mem3_sup.erl
+++ b/src/mem3_sup.erl
@@@ -22,10 -22,10 +22,11 @@@ start_link() -
init(_Args) ->
Children = [
child(mem3_events),
- child(mem3_cache),
child(mem3_nodes),
+ child(mem3_sync_nodes), % Order important?
child(mem3_sync),
- child(mem3_shards)
++ child(mem3_shards),
+ child(mem3_rep_manager)
],
{ok, {{one_for_one,10,1}, Children}}.
