couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dav...@apache.org
Subject [2/2] mem3 commit: updated refs/heads/COUCHDB-3287-pluggable-storage-engines to e2fa78f
Date Wed, 22 Mar 2017 21:31:07 GMT
Store and use the storage engine property

This adds an optional key to database documents that lists the
configured storage engine. This allows mem3_shards to create the shard
with the appropriate storage engine when recovering from a network
split.

COUCHDB-3287


Project: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/commit/0c9e63e1
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/tree/0c9e63e1
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/diff/0c9e63e1

Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: 0c9e63e1c564a3d5bfa6eef314639b3408a634cc
Parents: 6d00bae
Author: Paul J. Davis <paul.joseph.davis@gmail.com>
Authored: Wed Apr 6 11:18:01 2016 -0500
Committer: Paul J. Davis <paul.joseph.davis@gmail.com>
Committed: Wed Mar 22 16:31:00 2017 -0500

----------------------------------------------------------------------
 include/mem3.hrl        |  6 ++++--
 src/mem3.erl            | 20 +++++++++++++++---
 src/mem3_shards.erl     | 49 ++++++++++++++++----------------------------
 src/mem3_util.erl       | 17 ++++++++++++---
 test/mem3_util_test.erl | 16 +++++++--------
 5 files changed, 61 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/0c9e63e1/include/mem3.hrl
----------------------------------------------------------------------
diff --git a/include/mem3.hrl b/include/mem3.hrl
index d6ac0be..6579210 100644
--- a/include/mem3.hrl
+++ b/include/mem3.hrl
@@ -16,7 +16,8 @@
     node :: node() | '_',
     dbname :: binary(),
     range :: [non_neg_integer() | '$1' | '$2'] | '_',
-    ref :: reference() | 'undefined' | '_'
+    ref :: reference() | 'undefined' | '_',
+    opts :: list()
 }).
 
 %% Do not reference outside of mem3.
@@ -26,7 +27,8 @@
     dbname :: binary(),
     range :: [non_neg_integer() | '$1' | '$2'] | '_',
     ref :: reference() | 'undefined' | '_',
-    order :: non_neg_integer() | 'undefined' | '_'
+    order :: non_neg_integer() | 'undefined' | '_',
+    opts :: list()
 }).
 
 %% types

http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/0c9e63e1/src/mem3.erl
----------------------------------------------------------------------
diff --git a/src/mem3.erl b/src/mem3.erl
index e9c1473..5e218f7 100644
--- a/src/mem3.erl
+++ b/src/mem3.erl
@@ -23,7 +23,7 @@
 -export([get_placement/1]).
 
 %% For mem3 use only.
--export([name/1, node/1, range/1]).
+-export([name/1, node/1, range/1, engine/1]).
 
 -include_lib("mem3/include/mem3.hrl").
 -include_lib("couch/include/couch_db.hrl").
@@ -99,7 +99,8 @@ shards_int(DbName, Options) ->
             name = ShardDbName,
             dbname = ShardDbName,
             range = [0, (2 bsl 31)-1],
-            order = undefined}];
+            order = undefined,
+            opts = []}];
     ShardDbName ->
         %% shard_db is treated as a single sharded db to support calls to db_info
         %% and view_all_docs
@@ -107,7 +108,8 @@ shards_int(DbName, Options) ->
             node = node(),
             name = ShardDbName,
             dbname = ShardDbName,
-            range = [0, (2 bsl 31)-1]}];
+            range = [0, (2 bsl 31)-1],
+            opts = []}];
     _ ->
         mem3_shards:for_db(DbName, Options)
     end.
@@ -307,3 +309,15 @@ name(#shard{name=Name}) ->
     Name;
 name(#ordered_shard{name=Name}) ->
     Name.
+
+engine(#shard{opts=Opts}) ->
+    engine(Opts);
+engine(#ordered_shard{opts=Opts}) ->
+    engine(Opts);
+engine(Opts) when is_list(Opts) ->
+    case couch_util:get_value(engine, Opts) of
+        Engine when is_binary(Engine) ->
+            [{engine, Engine}];
+        _ ->
+            []
+    end.

http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/0c9e63e1/src/mem3_shards.erl
----------------------------------------------------------------------
diff --git a/src/mem3_shards.erl b/src/mem3_shards.erl
index a638db8..11131a1 100644
--- a/src/mem3_shards.erl
+++ b/src/mem3_shards.erl
@@ -65,19 +65,13 @@ for_docid(DbName, DocId) ->
 for_docid(DbName, DocId, Options) ->
     HashKey = mem3_util:hash(DocId),
     ShardHead = #shard{
-        name = '_',
-        node = '_',
         dbname = DbName,
-        range = ['$1','$2'],
-        ref = '_'
+        range = ['$1', '$2'],
+        _ = '_'
     },
     OrderedShardHead = #ordered_shard{
-        name = '_',
-        node = '_',
         dbname = DbName,
-        range = ['$1','$2'],
-        ref = '_',
-        order = '_'
+        _ = '_'
     },
     Conditions = [{'=<', '$1', HashKey}, {'=<', HashKey, '$2'}],
     ShardSpec = {ShardHead, Conditions, ['$_']},
@@ -103,18 +97,13 @@ for_shard_name(ShardName, Options) ->
     DbName = mem3:dbname(ShardName),
     ShardHead = #shard{
         name = ShardName,
-        node = '_',
         dbname = DbName,
-        range = '_',
-        ref = '_'
+        _ = '_'
     },
     OrderedShardHead = #ordered_shard{
         name = ShardName,
-        node = '_',
         dbname = DbName,
-        range = '_',
-        ref = '_',
-        order = '_'
+        _ = '_'
     },
     ShardSpec = {ShardHead, [], ['$_']},
     OrderedShardSpec = {OrderedShardHead, [], ['$_']},
@@ -300,7 +289,7 @@ changes_callback({change, {Change}, _}, _) ->
             {Doc} ->
                 Shards = mem3_util:build_ordered_shards(DbName, Doc),
                 gen_server:cast(?MODULE, {cache_insert, DbName, Shards}),
-                [create_if_missing(mem3:name(S)) || S
+                [create_if_missing(mem3:name(S), mem3:engine(S)) || S
                     <- Shards, mem3:node(S) =:= node()]
             end
         end
@@ -337,20 +326,18 @@ in_range(Shard, HashKey) ->
     [B, E] = mem3:range(Shard),
     B =< HashKey andalso HashKey =< E.
 
-create_if_missing(Name) ->
-    DbDir = config:get("couchdb", "database_dir"),
-    Filename = filename:join(DbDir, ?b2l(Name) ++ ".couch"),
-    case filelib:is_regular(Filename) of
-    true ->
-        ok;
-    false ->
-        case couch_server:create(Name, [?ADMIN_CTX]) of
-        {ok, Db} ->
-            couch_db:close(Db);
-        Error ->
-            couch_log:error("~p tried to create ~s, got ~p",
-                [?MODULE, Name, Error])
-        end
+create_if_missing(Name, Options) ->
+    case couch_server:exists(Name) of
+        true ->
+            ok;
+        false ->
+            case couch_server:create(Name, [?ADMIN_CTX] ++ Options) of
+            {ok, Db} ->
+                couch_db:close(Db);
+            Error ->
+                couch_log:error("~p tried to create ~s, got ~p",
+                    [?MODULE, Name, Error])
+            end
     end.
 
 cache_insert(#st{cur_size=Cur}=St, DbName, Shards) ->

http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/0c9e63e1/src/mem3_util.erl
----------------------------------------------------------------------
diff --git a/src/mem3_util.erl b/src/mem3_util.erl
index 2cd444d..4e1b5fd 100644
--- a/src/mem3_util.erl
+++ b/src/mem3_util.erl
@@ -153,6 +153,10 @@ build_ordered_shards(DbName, DocProps) ->
 build_shards_by_node(DbName, DocProps) ->
     {ByNode} = couch_util:get_value(<<"by_node">>, DocProps, {[]}),
     Suffix = couch_util:get_value(<<"shard_suffix">>, DocProps, ""),
+    EngineOpt = case couch_util:get_value(<<"engine">>, DocProps) of
+        Engine when is_binary(Engine) -> [{engine, Engine}];
+        _ -> []
+    end,
     lists:flatmap(fun({Node, Ranges}) ->
         lists:map(fun(Range) ->
             [B,E] = string:tokens(?b2l(Range), "-"),
@@ -161,7 +165,8 @@ build_shards_by_node(DbName, DocProps) ->
             name_shard(#shard{
                 dbname = DbName,
                 node = to_atom(Node),
-                range = [Beg, End]
+                range = [Beg, End],
+                opts = EngineOpt
             }, Suffix)
         end, Ranges)
     end, ByNode).
@@ -169,6 +174,10 @@ build_shards_by_node(DbName, DocProps) ->
 build_shards_by_range(DbName, DocProps) ->
     {ByRange} = couch_util:get_value(<<"by_range">>, DocProps, {[]}),
     Suffix = couch_util:get_value(<<"shard_suffix">>, DocProps, ""),
+    EngineOpt = case couch_util:get_value(<<"engine">>, DocProps) of
+        Engine when is_binary(Engine) -> [{engine, Engine}];
+        _ -> []
+    end,
     lists:flatmap(fun({Range, Nodes}) ->
         lists:map(fun({Node, Order}) ->
             [B,E] = string:tokens(?b2l(Range), "-"),
@@ -178,7 +187,8 @@ build_shards_by_range(DbName, DocProps) ->
                 dbname = DbName,
                 node = to_atom(Node),
                 range = [Beg, End],
-                order = Order
+                order = Order,
+                opts = EngineOpt
             }, Suffix)
         end, lists:zip(Nodes, lists:seq(1, length(Nodes))))
     end, ByRange).
@@ -247,7 +257,8 @@ downcast(#ordered_shard{}=S) ->
        node = S#ordered_shard.node,
        dbname = S#ordered_shard.dbname,
        range = S#ordered_shard.range,
-       ref = S#ordered_shard.ref
+       ref = S#ordered_shard.ref,
+       opts = S#ordered_shard.opts
       };
 downcast(Shards) when is_list(Shards) ->
     [downcast(Shard) || Shard <- Shards].

http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/0c9e63e1/test/mem3_util_test.erl
----------------------------------------------------------------------
diff --git a/test/mem3_util_test.erl b/test/mem3_util_test.erl
index 340a58a..42bc5c7 100644
--- a/test/mem3_util_test.erl
+++ b/test/mem3_util_test.erl
@@ -85,35 +85,35 @@ build_shards_test() ->
         [{shard,<<"shards/00000000-1fffffff/testdb1">>,
           'bigcouch@node.local',<<"testdb1">>,
           [0,536870911],
-          undefined},
+          undefined,[]},
          {shard,<<"shards/20000000-3fffffff/testdb1">>,
           'bigcouch@node.local',<<"testdb1">>,
           [536870912,1073741823],
-          undefined},
+          undefined,[]},
          {shard,<<"shards/40000000-5fffffff/testdb1">>,
           'bigcouch@node.local',<<"testdb1">>,
           [1073741824,1610612735],
-          undefined},
+          undefined,[]},
          {shard,<<"shards/60000000-7fffffff/testdb1">>,
           'bigcouch@node.local',<<"testdb1">>,
           [1610612736,2147483647],
-          undefined},
+          undefined,[]},
          {shard,<<"shards/80000000-9fffffff/testdb1">>,
           'bigcouch@node.local',<<"testdb1">>,
           [2147483648,2684354559],
-          undefined},
+          undefined,[]},
          {shard,<<"shards/a0000000-bfffffff/testdb1">>,
           'bigcouch@node.local',<<"testdb1">>,
           [2684354560,3221225471],
-          undefined},
+          undefined,[]},
          {shard,<<"shards/c0000000-dfffffff/testdb1">>,
           'bigcouch@node.local',<<"testdb1">>,
           [3221225472,3758096383],
-          undefined},
+          undefined,[]},
          {shard,<<"shards/e0000000-ffffffff/testdb1">>,
           'bigcouch@node.local',<<"testdb1">>,
           [3758096384,4294967295],
-          undefined}],
+          undefined,[]}],
     ?assertEqual(ExpectedShards1, Shards1),
     ok.
 


Mime
View raw message