Store and use the storage engine property This adds an optional key to database documents that lists the configured storage engine. This allows mem3_shards to create the shard with the appropriate storage engine when recovering from a network split.
COUCHDB-3287 Project: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/commit/925008ef Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/tree/925008ef Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/diff/925008ef Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines Commit: 925008efcf5f719ecd3c36bdbe3f2d6de57d0fd7 Parents: 6d00bae Author: Paul J. Davis <[email protected]> Authored: Wed Apr 6 11:18:01 2016 -0500 Committer: Paul J. Davis <[email protected]> Committed: Wed Mar 22 16:29:42 2017 -0500 ---------------------------------------------------------------------- include/mem3.hrl | 6 ++++-- src/mem3.erl | 20 +++++++++++++++--- src/mem3_shards.erl | 49 ++++++++++++++++---------------------------- src/mem3_util.erl | 17 ++++++++++++--- test/mem3_util_test.erl | 16 +++++++-------- 5 files changed, 61 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/925008ef/include/mem3.hrl ---------------------------------------------------------------------- diff --git a/include/mem3.hrl b/include/mem3.hrl index d6ac0be..6579210 100644 --- a/include/mem3.hrl +++ b/include/mem3.hrl @@ -16,7 +16,8 @@ node :: node() | '_', dbname :: binary(), range :: [non_neg_integer() | '$1' | '$2'] | '_', - ref :: reference() | 'undefined' | '_' + ref :: reference() | 'undefined' | '_', + opts :: list() }). %% Do not reference outside of mem3. @@ -26,7 +27,8 @@ dbname :: binary(), range :: [non_neg_integer() | '$1' | '$2'] | '_', ref :: reference() | 'undefined' | '_', - order :: non_neg_integer() | 'undefined' | '_' + order :: non_neg_integer() | 'undefined' | '_', + opts :: list() }). %% types http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/925008ef/src/mem3.erl ---------------------------------------------------------------------- diff --git a/src/mem3.erl b/src/mem3.erl index e9c1473..5e218f7 100644 --- a/src/mem3.erl +++ b/src/mem3.erl @@ -23,7 +23,7 @@ -export([get_placement/1]). %% For mem3 use only. --export([name/1, node/1, range/1]). +-export([name/1, node/1, range/1, engine/1]). -include_lib("mem3/include/mem3.hrl"). -include_lib("couch/include/couch_db.hrl"). @@ -99,7 +99,8 @@ shards_int(DbName, Options) -> name = ShardDbName, dbname = ShardDbName, range = [0, (2 bsl 31)-1], - order = undefined}]; + order = undefined, + opts = []}]; ShardDbName -> %% shard_db is treated as a single sharded db to support calls to db_info %% and view_all_docs @@ -107,7 +108,8 @@ shards_int(DbName, Options) -> node = node(), name = ShardDbName, dbname = ShardDbName, - range = [0, (2 bsl 31)-1]}]; + range = [0, (2 bsl 31)-1], + opts = []}]; _ -> mem3_shards:for_db(DbName, Options) end. @@ -307,3 +309,15 @@ name(#shard{name=Name}) -> Name; name(#ordered_shard{name=Name}) -> Name. + +engine(#shard{opts=Opts}) -> + engine(Opts); +engine(#ordered_shard{opts=Opts}) -> + engine(Opts); +engine(Opts) when is_list(Opts) -> + case couch_util:get_value(engine, Opts) of + Engine when is_binary(Engine) -> + [{engine, Engine}]; + _ -> + [] + end. http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/925008ef/src/mem3_shards.erl ---------------------------------------------------------------------- diff --git a/src/mem3_shards.erl b/src/mem3_shards.erl index a638db8..0eb42e5 100644 --- a/src/mem3_shards.erl +++ b/src/mem3_shards.erl @@ -65,19 +65,13 @@ for_docid(DbName, DocId) -> for_docid(DbName, DocId, Options) -> HashKey = mem3_util:hash(DocId), ShardHead = #shard{ - name = '_', - node = '_', dbname = DbName, - range = ['$1','$2'], - ref = '_' + range = ['$1', '$2'}, + _ = '_' }, OrderedShardHead = #ordered_shard{ - name = '_', - node = '_', dbname = DbName, - range = ['$1','$2'], - ref = '_', - order = '_' + _ = '_' }, Conditions = [{'=<', '$1', HashKey}, {'=<', HashKey, '$2'}], ShardSpec = {ShardHead, Conditions, ['$_']}, @@ -103,18 +97,13 @@ for_shard_name(ShardName, Options) -> DbName = mem3:dbname(ShardName), ShardHead = #shard{ name = ShardName, - node = '_', dbname = DbName, - range = '_', - ref = '_' + _ = '_' }, OrderedShardHead = #ordered_shard{ name = ShardName, - node = '_', dbname = DbName, - range = '_', - ref = '_', - order = '_' + _ = '_' }, ShardSpec = {ShardHead, [], ['$_']}, OrderedShardSpec = {OrderedShardHead, [], ['$_']}, @@ -300,7 +289,7 @@ changes_callback({change, {Change}, _}, _) -> {Doc} -> Shards = mem3_util:build_ordered_shards(DbName, Doc), gen_server:cast(?MODULE, {cache_insert, DbName, Shards}), - [create_if_missing(mem3:name(S)) || S + [create_if_missing(mem3:name(S), mem3:engine(S)) || S <- Shards, mem3:node(S) =:= node()] end end @@ -337,20 +326,18 @@ in_range(Shard, HashKey) -> [B, E] = mem3:range(Shard), B =< HashKey andalso HashKey =< E. -create_if_missing(Name) -> - DbDir = config:get("couchdb", "database_dir"), - Filename = filename:join(DbDir, ?b2l(Name) ++ ".couch"), - case filelib:is_regular(Filename) of - true -> - ok; - false -> - case couch_server:create(Name, [?ADMIN_CTX]) of - {ok, Db} -> - couch_db:close(Db); - Error -> - couch_log:error("~p tried to create ~s, got ~p", - [?MODULE, Name, Error]) - end +create_if_missing(Name, Options) -> + case couch_server:exists(Name) of + true -> + ok; + false -> + case couch_server:create(Name, [?ADMIN_CTX] ++ Options) of + {ok, Db} -> + couch_db:close(Db); + Error -> + couch_log:error("~p tried to create ~s, got ~p", + [?MODULE, Name, Error]) + end end. cache_insert(#st{cur_size=Cur}=St, DbName, Shards) -> http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/925008ef/src/mem3_util.erl ---------------------------------------------------------------------- diff --git a/src/mem3_util.erl b/src/mem3_util.erl index 2cd444d..4e1b5fd 100644 --- a/src/mem3_util.erl +++ b/src/mem3_util.erl @@ -153,6 +153,10 @@ build_ordered_shards(DbName, DocProps) -> build_shards_by_node(DbName, DocProps) -> {ByNode} = couch_util:get_value(<<"by_node">>, DocProps, {[]}), Suffix = couch_util:get_value(<<"shard_suffix">>, DocProps, ""), + EngineOpt = case couch_util:get_value(<<"engine">>, DocProps) of + Engine when is_binary(Engine) -> [{engine, Engine}]; + _ -> [] + end, lists:flatmap(fun({Node, Ranges}) -> lists:map(fun(Range) -> [B,E] = string:tokens(?b2l(Range), "-"), @@ -161,7 +165,8 @@ build_shards_by_node(DbName, DocProps) -> name_shard(#shard{ dbname = DbName, node = to_atom(Node), - range = [Beg, End] + range = [Beg, End], + opts = EngineOpt }, Suffix) end, Ranges) end, ByNode). @@ -169,6 +174,10 @@ build_shards_by_node(DbName, DocProps) -> build_shards_by_range(DbName, DocProps) -> {ByRange} = couch_util:get_value(<<"by_range">>, DocProps, {[]}), Suffix = couch_util:get_value(<<"shard_suffix">>, DocProps, ""), + EngineOpt = case couch_util:get_value(<<"engine">>, DocProps) of + Engine when is_binary(Engine) -> [{engine, Engine}]; + _ -> [] + end, lists:flatmap(fun({Range, Nodes}) -> lists:map(fun({Node, Order}) -> [B,E] = string:tokens(?b2l(Range), "-"), @@ -178,7 +187,8 @@ build_shards_by_range(DbName, DocProps) -> dbname = DbName, node = to_atom(Node), range = [Beg, End], - order = Order + order = Order, + opts = EngineOpt }, Suffix) end, lists:zip(Nodes, lists:seq(1, length(Nodes)))) end, ByRange). @@ -247,7 +257,8 @@ downcast(#ordered_shard{}=S) -> node = S#ordered_shard.node, dbname = S#ordered_shard.dbname, range = S#ordered_shard.range, - ref = S#ordered_shard.ref + ref = S#ordered_shard.ref, + opts = S#ordered_shard.opts }; downcast(Shards) when is_list(Shards) -> [downcast(Shard) || Shard <- Shards]. http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/925008ef/test/mem3_util_test.erl ---------------------------------------------------------------------- diff --git a/test/mem3_util_test.erl b/test/mem3_util_test.erl index 340a58a..42bc5c7 100644 --- a/test/mem3_util_test.erl +++ b/test/mem3_util_test.erl @@ -85,35 +85,35 @@ build_shards_test() -> [{shard,<<"shards/00000000-1fffffff/testdb1">>, '[email protected]',<<"testdb1">>, [0,536870911], - undefined}, + undefined,[]}, {shard,<<"shards/20000000-3fffffff/testdb1">>, '[email protected]',<<"testdb1">>, [536870912,1073741823], - undefined}, + undefined,[]}, {shard,<<"shards/40000000-5fffffff/testdb1">>, '[email protected]',<<"testdb1">>, [1073741824,1610612735], - undefined}, + undefined,[]}, {shard,<<"shards/60000000-7fffffff/testdb1">>, '[email protected]',<<"testdb1">>, [1610612736,2147483647], - undefined}, + undefined,[]}, {shard,<<"shards/80000000-9fffffff/testdb1">>, '[email protected]',<<"testdb1">>, [2147483648,2684354559], - undefined}, + undefined,[]}, {shard,<<"shards/a0000000-bfffffff/testdb1">>, '[email protected]',<<"testdb1">>, [2684354560,3221225471], - undefined}, + undefined,[]}, {shard,<<"shards/c0000000-dfffffff/testdb1">>, '[email protected]',<<"testdb1">>, [3221225472,3758096383], - undefined}, + undefined,[]}, {shard,<<"shards/e0000000-ffffffff/testdb1">>, '[email protected]',<<"testdb1">>, [3758096384,4294967295], - undefined}], + undefined,[]}], ?assertEqual(ExpectedShards1, Shards1), ok.
