couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rnew...@apache.org
Subject [4/4] mem3 commit: updated refs/heads/master to 0b70afb
Date Mon, 09 May 2016 21:29:21 GMT
Use ets:select/2 to retrieve shards by name

The result of mem3_shards:for_db/1 on databases with high q values can
be very large, resulting in suboptimal performance for high-volume
callers.

mem3_sync_event_listener is only interested in a small subset of the
result of mem3_shards:for_db/1; moving this filter in to an ets:select/2
call improves performance significantly.

COUCHDB-2984


Project: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/commit/130efcd6
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/tree/130efcd6
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/diff/130efcd6

Branch: refs/heads/master
Commit: 130efcd6b0b6f9fa6403e131586dfbf003643074
Parents: d5e0a4a
Author: Benjamin Anderson <b@banjiewen.net>
Authored: Sat Apr 9 23:08:39 2016 -0700
Committer: Benjamin Anderson <b@banjiewen.net>
Committed: Mon May 9 14:20:56 2016 -0700

----------------------------------------------------------------------
 src/mem3_shards.erl              | 49 +++++++++++++++++++++++++++++++++++
 src/mem3_sync_event_listener.erl | 15 +++++++----
 2 files changed, 59 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/130efcd6/src/mem3_shards.erl
----------------------------------------------------------------------
diff --git a/src/mem3_shards.erl b/src/mem3_shards.erl
index 14aed34..224673f 100644
--- a/src/mem3_shards.erl
+++ b/src/mem3_shards.erl
@@ -21,6 +21,7 @@
 
 -export([start_link/0]).
 -export([for_db/1, for_db/2, for_docid/2, for_docid/3, get/3, local/1, fold/2]).
+-export([for_shard_name/1]).
 -export([set_max_size/1]).
 
 -record(st, {
@@ -94,6 +95,42 @@ for_docid(DbName, DocId, Options) ->
         false -> mem3_util:downcast(Shards)
     end.
 
+for_shard_name(ShardName) ->
+    for_shard_name(ShardName, []).
+
+for_shard_name(ShardName, Options) ->
+    DbName = mem3:dbname(ShardName),
+    ShardHead = #shard{
+        name = ShardName,
+        node = '_',
+        dbname = DbName,
+        range = '_',
+        ref = '_'
+    },
+    OrderedShardHead = #ordered_shard{
+        name = ShardName,
+        node = '_',
+        dbname = DbName,
+        range = '_',
+        ref = '_',
+        order = '_'
+    },
+    ShardSpec = {ShardHead, [], ['$_']},
+    OrderedShardSpec = {OrderedShardHead, [], ['$_']},
+    Shards = try ets:select(?SHARDS, [ShardSpec, OrderedShardSpec]) of
+        [] ->
+            filter_shards_by_name(ShardName, load_shards_from_disk(DbName));
+        Else ->
+            gen_server:cast(?MODULE, {cache_hit, DbName}),
+            Else
+    catch error:badarg ->
+        filter_shards_by_name(ShardName, load_shards_from_disk(DbName))
+    end,
+    case lists:member(ordered, Options) of
+        true  -> Shards;
+        false -> mem3_util:downcast(Shards)
+    end.
+
 get(DbName, Node, Range) ->
     Res = lists:foldl(fun(#shard{node=N, range=R}=S, Acc) ->
         case {N, R} of
@@ -360,3 +397,15 @@ cache_clear(St) ->
     true = ets:delete_all_objects(?SHARDS),
     true = ets:delete_all_objects(?ATIMES),
     St#st{cur_size=0}.
+
+filter_shards_by_name(Name, Shards) ->
+    filter_shards_by_name(Name, [], Shards).
+
+filter_shards_by_name(_, Matches, []) ->
+    Matches;
+filter_shards_by_name(Name, Matches, [#ordered_shard{name=Name}=S|Ss]) ->
+    filter_shards_by_name(Name, [S|Matches], Ss);
+filter_shards_by_name(Name, Matches, [#shard{name=Name}=S|Ss]) ->
+    filter_shards_by_name(Name, [S|Matches], Ss);
+filter_shards_by_name(Name, Matches, [_|Ss]) ->
+    filter_shards_by_name(Name, Matches, Ss).

http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/130efcd6/src/mem3_sync_event_listener.erl
----------------------------------------------------------------------
diff --git a/src/mem3_sync_event_listener.erl b/src/mem3_sync_event_listener.erl
index d74c21f..ca058db 100644
--- a/src/mem3_sync_event_listener.erl
+++ b/src/mem3_sync_event_listener.erl
@@ -207,13 +207,18 @@ maybe_push_shards(St) ->
     end.
 
 push_shard(ShardName) ->
-    try mem3:shards(mem3:dbname(ShardName)) of
+    try mem3_shards:for_shard_name(ShardName) of
     Shards ->
-        Targets = [S || #shard{node=N, name=Name} = S <- Shards,
-            N =/= node(), Name =:= ShardName],
         Live = nodes(),
-        [mem3_sync:push(ShardName,N) || #shard{node=N} <- Targets,
-            lists:member(N, Live)]
+        lists:foreach(
+            fun(#shard{node=N}) ->
+                case lists:member(N, Live) of
+                    true -> mem3_sync:push(ShardName, N);
+                    false -> ok
+                end
+            end,
+            Shards
+        )
     catch error:database_does_not_exist ->
         ok
     end.


Mime
View raw message