couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dav...@apache.org
Subject [19/36] couch commit: updated refs/heads/COUCHDB-3287-pluggable-storage-engines to 0fbcd0c
Date Thu, 06 Apr 2017 16:16:33 GMT
Implement an ETS-basd couch_lru

Use a monotonicaly incrementing counter instead of `erlang:now()`. We don't
technically need to time-based functionality and just want to know relative
insertion order.

Instead of gb_tree, use an ordered_set ETS. This keep items sorted by their
update order, with most recent ones at the bottom.

An set ETS replaces the dictionary which maintains a mapping from database
names to their entry in updates table.

Interface is the same as the old couch_lru, so it is a direct swap in.

Thanks to Eric Avdey for intial version of test module.

COUCHDB-3326


Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/016e1aa0
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/016e1aa0
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/016e1aa0

Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: 016e1aa0ef4db0bbf47a28a2cce48b85200702d6
Parents: c50a8cd
Author: Nick Vatamaniuc <vatamane@apache.org>
Authored: Wed Mar 15 14:06:25 2017 -0400
Committer: Benjamin Bastian <benjamin.bastian@gmail.com>
Committed: Fri Mar 17 12:16:40 2017 -0700

----------------------------------------------------------------------
 src/couch_lru.erl        |  60 +++++++++++++----------
 test/couch_lru_tests.erl | 109 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 143 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/016e1aa0/src/couch_lru.erl
----------------------------------------------------------------------
diff --git a/src/couch_lru.erl b/src/couch_lru.erl
index 84d7bee..b79286e 100644
--- a/src/couch_lru.erl
+++ b/src/couch_lru.erl
@@ -16,33 +16,39 @@
 -include_lib("couch/include/couch_db.hrl").
 
 new() ->
-    {gb_trees:empty(), dict:new()}.
-
-insert(DbName, {Tree0, Dict0}) ->
-    Lru = erlang:now(),
-    {gb_trees:insert(Lru, DbName, Tree0), dict:store(DbName, Lru, Dict0)}.
-
-update(DbName, {Tree0, Dict0}) ->
-    case dict:find(DbName, Dict0) of
-    {ok, Old} ->
-        New = erlang:now(),
-        Tree = gb_trees:insert(New, DbName, gb_trees:delete(Old, Tree0)),
-        Dict = dict:store(DbName, New, Dict0),
-        {Tree, Dict};
-    error ->
-        % We closed this database before processing the update.  Ignore
-        {Tree0, Dict0}
+    Updates = ets:new(couch_lru_updates, [ordered_set]),
+    Dbs = ets:new(couch_lru_dbs, [set]),
+    {0, Updates, Dbs}.
+
+insert(DbName, {Count, Updates, Dbs}) ->
+    update(DbName, {Count, Updates, Dbs}).
+
+update(DbName, {Count, Updates, Dbs}) ->
+    case ets:lookup(Dbs, DbName) of
+        [] ->
+            true = ets:insert(Dbs, {DbName, Count});
+        [{DbName, OldCount}] ->
+            true = ets:update_element(Dbs, DbName, {2, Count}),
+            true = ets:delete(Updates, {OldCount, DbName})
+    end,
+    true = ets:insert(Updates, {{Count, DbName}}),
+    {Count + 1, Updates, Dbs}.
+
+
+close({Count, Updates, Dbs}) ->
+    case close_int(ets:next(Updates, {-1, <<>>}), Updates, Dbs) of
+        true ->
+            {true, {Count, Updates, Dbs}};
+        false ->
+            false
     end.
 
-%% Attempt to close the oldest idle database.
-close({Tree, _} = Cache) ->
-    close_int(gb_trees:next(gb_trees:iterator(Tree)), Cache).
 
 %% internals
 
-close_int(none, _) ->
+close_int('$end_of_table', _Updates, _Dbs) ->
     false;
-close_int({Lru, DbName, Iter}, {Tree, Dict} = Cache) ->
+close_int({_Count, DbName} = Key, Updates, Dbs) ->
     case ets:update_element(couch_dbs, DbName, {#db.fd_monitor, locked}) of
     true ->
         [#db{main_pid = Pid} = Db] = ets:lookup(couch_dbs, DbName),
@@ -50,14 +56,16 @@ close_int({Lru, DbName, Iter}, {Tree, Dict} = Cache) ->
             true = ets:delete(couch_dbs, DbName),
             true = ets:delete(couch_dbs_pid_to_name, Pid),
             exit(Pid, kill),
-            {true, {gb_trees:delete(Lru, Tree), dict:erase(DbName, Dict)}};
+            true = ets:delete(Updates, Key),
+            true = ets:delete(Dbs, DbName),
+            true;
         false ->
             true = ets:update_element(couch_dbs, DbName, {#db.fd_monitor, nil}),
             couch_stats:increment_counter([couchdb, couch_server, lru_skip]),
-            close_int(gb_trees:next(Iter), update(DbName, Cache))
+            close_int(ets:next(Updates, Key), Updates, Dbs)
         end;
     false ->
-        NewTree = gb_trees:delete(Lru, Tree),
-        NewIter = gb_trees:iterator(NewTree),
-        close_int(gb_trees:next(NewIter), {NewTree, dict:erase(DbName, Dict)})
+        true = ets:delete(Updates, Key),
+        true = ets:delete(Dbs, DbName),
+        close_int(ets:next(Updates, Key), Updates, Dbs)
     end.

http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/016e1aa0/test/couch_lru_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_lru_tests.erl b/test/couch_lru_tests.erl
new file mode 100644
index 0000000..1559835
--- /dev/null
+++ b/test/couch_lru_tests.erl
@@ -0,0 +1,109 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_lru_tests).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+
+setup() ->
+    ok = meck:new(couch_db, [passthrough]),
+    ets:new(couch_dbs, [set, public, named_table, {keypos, #db.name}]),
+    ets:new(couch_dbs_pid_to_name, [set, public, named_table]),
+    couch_lru:new().
+
+teardown(_) ->
+    ets:delete(couch_dbs),
+    ets:delete(couch_dbs_pid_to_name),
+    (catch meck:unload(couch_db)).
+
+new_test_() ->
+    {setup,
+        fun() -> couch_lru:new() end,
+        fun(Lru) ->
+            ?_assertMatch({0, _, _}, Lru)
+        end
+    }.
+
+insert_test_() ->
+    {setup,
+        fun() -> couch_lru:new() end,
+        fun(Lru) ->
+            Key = <<"test">>,
+            {1, Updates, Dbs} = couch_lru:insert(Key, Lru),
+            [
+                ?_assertEqual(1, ets_size(Dbs)),
+                ?_assert(ets:member(Dbs, Key)),
+                ?_assertEqual(1, ets_size(Updates)),
+                ?_assert(ets:member(Updates, {0, Key}))
+            ]
+        end
+    }.
+
+insert_same_test_() ->
+    {setup,
+        fun() -> couch_lru:new() end,
+        fun(Lru) ->
+            Key = <<"test">>,
+            Lru1 = {1, Updates, Dbs} = couch_lru:insert(Key, Lru),
+            {2, Updates, Dbs} = couch_lru:insert(Key, Lru1),
+            [
+                ?_assertEqual(1, ets_size(Dbs)),
+                ?_assert(ets:member(Dbs, Key)),
+                ?_assertEqual(1, ets_size(Updates)),
+                ?_assert(ets:member(Updates, {1, Key}))
+            ]
+        end
+    }.
+
+update_test_() ->
+    {setup,
+        fun() -> couch_lru:new() end,
+        fun(Lru) ->
+            Key = <<"test">>,
+            Lru1 = {1, Updates, Dbs} = couch_lru:update(Key, Lru),
+            {2, Updates, Dbs} = couch_lru:update(Key, Lru1),
+            [
+                ?_assertEqual(1, ets_size(Dbs)),
+                ?_assert(ets:member(Dbs, Key)),
+                ?_assertEqual(1, ets_size(Updates)),
+                ?_assert(ets:member(Updates, {1, Key}))
+            ]
+        end
+    }.
+
+close_test_() ->
+    {setup,
+        fun setup/0,
+        fun teardown/1,
+        fun(Lru) ->
+            ok = meck:expect(couch_db, is_idle, 1, true),
+            {ok, Lru1} = add_record(Lru, <<"test1">>, c:pid(0, 1001, 0)),
+            {ok, Lru2} = add_record(Lru1, <<"test2">>, c:pid(0, 2001, 0)),
+            {true, {2, Updates, Dbs}} = couch_lru:close(Lru2),
+            [
+                ?_assertEqual(1, ets_size(Dbs)),
+                ?_assert(ets:member(Dbs, <<"test2">>)),
+                ?_assertEqual(1, ets_size(Updates)),
+                ?_assert(ets:member(Updates, {1, <<"test2">>}))
+            ]
+        end
+    }.
+
+add_record(Lru, Key, Pid) ->
+    true = ets:insert(couch_dbs, #db{name = Key, main_pid = Pid}),
+    true = ets:insert(couch_dbs_pid_to_name, {Pid, Key}),
+    {ok, couch_lru:insert(Key, Lru)}.
+
+ets_size(Ets) ->
+    proplists:get_value(size, ets:info(Ets)).


Mime
View raw message