couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dav...@apache.org
Subject [couchdb] branch optimize-ddoc-cache updated: SQUAHS: Test suite updates
Date Thu, 29 Jun 2017 21:11:19 GMT
This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch optimize-ddoc-cache
in repository https://gitbox.apache.org/repos/asf/couchdb.git


The following commit(s) were added to refs/heads/optimize-ddoc-cache by this push:
     new d022f9a  SQUAHS: Test suite updates
d022f9a is described below

commit d022f9af725386289aad1301677c087207861f3b
Author: Paul J. Davis <paul.joseph.davis@gmail.com>
AuthorDate: Thu Jun 29 16:10:58 2017 -0500

    SQUAHS: Test suite updates
---
 src/ddoc_cache/src/ddoc_cache_entry.erl          |   9 +-
 src/ddoc_cache/src/ddoc_cache_lru.erl            |  30 +++--
 src/ddoc_cache/test/ddoc_cache_coverage_test.erl |   6 +-
 src/ddoc_cache/test/ddoc_cache_entry_test.erl    | 120 ++++++++++++++++++
 src/ddoc_cache/test/ddoc_cache_eviction_test.erl |   2 +
 src/ddoc_cache/test/ddoc_cache_lru_test.erl      | 153 +++++++++++++++++++++++
 src/ddoc_cache/test/ddoc_cache_tutil.erl         |   2 +-
 7 files changed, 300 insertions(+), 22 deletions(-)

diff --git a/src/ddoc_cache/src/ddoc_cache_entry.erl b/src/ddoc_cache/src/ddoc_cache_entry.erl
index 4689390..4dee2a1 100644
--- a/src/ddoc_cache/src/ddoc_cache_entry.erl
+++ b/src/ddoc_cache/src/ddoc_cache_entry.erl
@@ -98,17 +98,13 @@ refresh(Pid) ->
 
 
 init(Key) ->
-    Entry = #entry{
-        key = Key,
-        val = undefined,
-        pid = self()
-    },
-    true = ets:insert(?CACHE, Entry),
+    true = ets:update_element(?CACHE, Key, {#entry.pid, self()}),
     St = #st{
         key = Key,
         opener = spawn_opener(Key),
         waiters = []
     },
+    ?EVENT(started, Key),
     gen_server:enter_loop(?MODULE, [], St).
 
 
@@ -281,4 +277,3 @@ drain_accessed() ->
 
 respond(Waiters, Resp) ->
     [gen_server:reply(W, Resp) || W <- Waiters].
-
diff --git a/src/ddoc_cache/src/ddoc_cache_lru.erl b/src/ddoc_cache/src/ddoc_cache_lru.erl
index 5988f03..14440f2 100644
--- a/src/ddoc_cache/src/ddoc_cache_lru.erl
+++ b/src/ddoc_cache/src/ddoc_cache_lru.erl
@@ -69,8 +69,8 @@ open(Key) ->
             ddoc_cache_entry:accessed(Pid),
             {ok, Val}
     catch _:_ ->
-            couch_stats:increment_counter([ddoc_cache, recovery]),
-            ddoc_cache_entry:recover(Key)
+        couch_stats:increment_counter([ddoc_cache, recovery]),
+        ddoc_cache_entry:recover(Key)
     end.
 
 
@@ -107,15 +107,23 @@ handle_call({start, Key}, _From, St) ->
         dbs = Dbs,
         size = CurSize
     } = St,
-    MaxSize = max(0, config:get_integer("ddoc_cache", "max_size", 500)),
-    case trim(St, CurSize, MaxSize) of
-        {ok, N} ->
-            {ok, Pid} = ddoc_cache_entry:start_link(Key),
-            ok = khash:put(Pids, Pid, Key),
-            store_key(Dbs, Key, Pid),
-            {reply, {ok, Pid}, St#st{size = CurSize - N + 1}};
-        full ->
-            {reply, full, St}
+    case ets:lookup(?CACHE, Key) of
+        [] ->
+            MaxSize = config:get_integer("ddoc_cache", "max_size", 1000),
+            case trim(St, CurSize, max(0, MaxSize)) of
+                {ok, N} ->
+                    true = ets:insert_new(?CACHE, #entry{key = Key}),
+                    {ok, Pid} = ddoc_cache_entry:start_link(Key),
+                    true = ets:update_element(?CACHE, Key, {#entry.pid, Pid}),
+                    ok = khash:put(Pids, Pid, Key),
+                    store_key(Dbs, Key, Pid),
+                    {reply, {ok, Pid}, St#st{size = CurSize - N + 1}};
+                full ->
+                    ?EVENT(full, Key),
+                    {reply, full, St}
+            end;
+        [#entry{pid = Pid}] ->
+            {reply, {ok, Pid}, St}
     end;
 
 handle_call(Msg, _From, St) ->
diff --git a/src/ddoc_cache/test/ddoc_cache_coverage_test.erl b/src/ddoc_cache/test/ddoc_cache_coverage_test.erl
index 91182ca..395f560 100644
--- a/src/ddoc_cache/test/ddoc_cache_coverage_test.erl
+++ b/src/ddoc_cache/test/ddoc_cache_coverage_test.erl
@@ -33,7 +33,7 @@ coverage_test_() ->
 
 restart_lru() ->
     send_bad_messages(ddoc_cache_lru),
-    ?assertEqual(ok, ddoc_cache_lru:terminate(bang, {st, a, b, c})),
+    ?assertEqual(ok, ddoc_cache_lru:terminate(bang, {st, a, b, c, d})),
     ?assertEqual({ok, foo}, ddoc_cache_lru:code_change(1, foo, [])).
 
 
@@ -47,7 +47,7 @@ restart_evictor() ->
     meck:new(ddoc_cache_ev, [passthrough]),
     try
         State = sys:get_state(ddoc_cache_lru),
-        Evictor = element(4, State),
+        Evictor = element(5, State),
         Ref = erlang:monitor(process, Evictor),
         exit(Evictor, shutdown),
         receive
@@ -57,7 +57,7 @@ restart_evictor() ->
         end,
         meck:wait(ddoc_cache_ev, event, [evictor_died, '_'], 1000),
         NewState = sys:get_state(ddoc_cache_lru),
-        NewEvictor = element(4, NewState),
+        NewEvictor = element(5, NewState),
         ?assertNotEqual(Evictor, NewEvictor)
     after
         meck:unload()
diff --git a/src/ddoc_cache/test/ddoc_cache_entry_test.erl b/src/ddoc_cache/test/ddoc_cache_entry_test.erl
new file mode 100644
index 0000000..62afc72
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_entry_test.erl
@@ -0,0 +1,120 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_entry_test).
+
+
+-export([
+    recover/1
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+recover(<<"foo">>) ->
+    timer:sleep(30000);
+
+recover(DbName) ->
+    {ok, {DbName, such_custom}}.
+
+
+start_couch() ->
+    Ctx = ddoc_cache_tutil:start_couch(),
+    meck:new(ddoc_cache_ev, [passthrough]),
+    Ctx.
+
+
+stop_couch(Ctx) ->
+    meck:unload(),
+    ddoc_cache_tutil:stop_couch(Ctx).
+
+
+check_entry_test_() ->
+    {
+        setup,
+        fun start_couch/0,
+        fun stop_couch/1,
+        {with, [
+            fun cancel_and_replace_opener/1,
+            fun condenses_access_messages/1,
+            fun kill_opener_on_terminate/1,
+            fun open_dead_entry/1,
+            fun handles_bad_messages/1,
+            fun handles_code_change/1
+        ]}
+    }.
+
+
+cancel_and_replace_opener(_) ->
+    Key = {ddoc_cache_entry_custom, {<<"foo">>, ?MODULE}},
+    true = ets:insert_new(?CACHE, #entry{key = Key}),
+    {ok, Entry} = ddoc_cache_entry:start_link(Key),
+    Opener1 = element(4, sys:get_state(Entry)),
+    Ref1 = erlang:monitor(process, Opener1),
+    gen_server:cast(Entry, refresh),
+    receive {'DOWN', Ref1, _, _, _} -> ok end,
+    Opener2 = element(4, sys:get_state(Entry)),
+    ?assert(Opener2 /= Opener1),
+    ?assert(is_process_alive(Opener2)),
+    % Clean up after ourselves
+    unlink(Entry),
+    ddoc_cache_entry:shutdown(Entry).
+
+
+condenses_access_messages({DbName, _}) ->
+    meck:reset(ddoc_cache_ev),
+    Key = {ddoc_cache_entry_custom, {DbName, ?MODULE}},
+    true = ets:insert(?CACHE, #entry{key = Key}),
+    {ok, Entry} = ddoc_cache_entry:start_link(Key),
+    erlang:suspend_process(Entry),
+    lists:foreach(fun(_) ->
+        gen_server:cast(Entry, accessed)
+    end, lists:seq(1, 100)),
+    erlang:resume_process(Entry),
+    meck:wait(1, ddoc_cache_ev, event, [accessed, Key], 1000),
+    ?assertError(
+            timeout,
+            meck:wait(2, ddoc_cache_ev, event, [accessed, Key], 100)
+        ),
+    unlink(Entry),
+    ddoc_cache_entry:shutdown(Entry).
+
+
+kill_opener_on_terminate(_) ->
+    Pid = spawn(fun() -> receive _ -> ok end end),
+    ?assert(is_process_alive(Pid)),
+    St = {st, key, val, Pid, waiters, ts},
+    ?assertEqual(ok, ddoc_cache_entry:terminate(normal, St)),
+    ?assert(not is_process_alive(Pid)).
+
+
+open_dead_entry({DbName, _}) ->
+    Pid = spawn(fun() -> ok end),
+    Key = {ddoc_cache_entry_custom, {DbName, ?MODULE}},
+    ?assertEqual(recover(DbName), ddoc_cache_entry:open(Pid, Key)).
+
+
+handles_bad_messages(_) ->
+    CallExpect = {stop, {bad_call, foo}, {bad_call, foo}, baz},
+    CastExpect = {stop, {bad_cast, foo}, bar},
+    InfoExpect = {stop, {bad_info, foo}, bar},
+    ?assertEqual(CallExpect, ddoc_cache_entry:handle_call(foo, bar, baz)),
+    ?assertEqual(CastExpect, ddoc_cache_entry:handle_cast(foo, bar)),
+    ?assertEqual(InfoExpect, ddoc_cache_entry:handle_info(foo, bar)).
+
+
+handles_code_change(_) ->
+    CCExpect = {ok, bar},
+    ?assertEqual(CCExpect, ddoc_cache_entry:code_change(foo, bar, baz)).
diff --git a/src/ddoc_cache/test/ddoc_cache_eviction_test.erl b/src/ddoc_cache/test/ddoc_cache_eviction_test.erl
index 62ed0e8..0b9f57b 100644
--- a/src/ddoc_cache/test/ddoc_cache_eviction_test.erl
+++ b/src/ddoc_cache/test/ddoc_cache_eviction_test.erl
@@ -63,6 +63,7 @@ evict_all({DbName, _}) ->
     ?assertEqual(4, ets:info(?CACHE, size)),
     {ok, _} = ddoc_cache_lru:handle_db_event(ShardName, deleted, foo),
     meck:wait(ddoc_cache_ev, event, [evicted, DbName], 1000),
+    meck:wait(4, ddoc_cache_ev, event, [removed, '_'], 1000),
     ?assertEqual(0, ets:info(?CACHE, size)).
 
 
@@ -88,4 +89,5 @@ check_upgrade_clause({DbName, _}) ->
     ?assertEqual(1, ets:info(?CACHE, size)),
     gen_server:cast(ddoc_cache_opener, {do_evict, DbName}),
     meck:wait(ddoc_cache_ev, event, [evicted, DbName], 1000),
+    meck:wait(ddoc_cache_ev, event, [removed, '_'], 1000),
     ?assertEqual(0, ets:info(?CACHE, size)).
diff --git a/src/ddoc_cache/test/ddoc_cache_lru_test.erl b/src/ddoc_cache/test/ddoc_cache_lru_test.erl
new file mode 100644
index 0000000..c3058ff
--- /dev/null
+++ b/src/ddoc_cache/test/ddoc_cache_lru_test.erl
@@ -0,0 +1,153 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(ddoc_cache_lru_test).
+
+
+-export([
+    recover/1
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include("ddoc_cache_test.hrl").
+
+
+recover(<<"pause", _/binary>>) ->
+    receive go -> ok end,
+    {ok, paused};
+
+recover(DbName) ->
+    {ok, DbName}.
+
+
+start_couch() ->
+    Ctx = ddoc_cache_tutil:start_couch(),
+    config:set("ddoc_cache", "max_size", "5", false),
+    meck:new(ddoc_cache_ev, [passthrough]),
+    Ctx.
+
+
+stop_couch(Ctx) ->
+    meck:unload(),
+    ddoc_cache_tutil:stop_couch(Ctx).
+
+
+check_not_started_test() ->
+    Key = {ddoc_cache_entry_custom, {<<"dbname">>, ?MODULE}},
+    ?assertEqual({ok, <<"dbname">>}, ddoc_cache_lru:open(Key)).
+
+
+check_lru_test_() ->
+    {
+        setup,
+        fun start_couch/0,
+        fun stop_couch/1,
+        {with, [
+            fun check_multi_start/1,
+            fun check_multi_open/1,
+            fun check_capped_size/1,
+            fun check_full_cache/1
+        ]}
+    }.
+
+
+check_multi_start(_) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    Key = {ddoc_cache_entry_custom, {<<"pause">>, ?MODULE}},
+    % These will all get sent through ddoc_cache_lru
+    Clients = lists:map(fun(_) ->
+        spawn_monitor(fun() ->
+            ddoc_cache_lru:open(Key)
+        end)
+    end, lists:seq(1, 10)),
+    meck:wait(ddoc_cache_ev, event, [started, Key], 1000),
+    lists:foreach(fun({Pid, _Ref}) ->
+        ?assert(is_process_alive(Pid))
+    end, Clients),
+    [#entry{pid = Pid}] = ets:tab2list(?CACHE),
+    Opener = element(4, sys:get_state(Pid)),
+    OpenerRef = erlang:monitor(process, Opener),
+    ?assert(is_process_alive(Opener)),
+    Opener ! go,
+    receive {'DOWN', OpenerRef, _, _, _} -> ok end,
+    lists:foreach(fun({_, Ref}) ->
+        receive {'DOWN', Ref, _, _, normal} -> ok end
+    end, Clients).
+
+
+check_multi_open(_) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    Key = {ddoc_cache_entry_custom, {<<"pause">>, ?MODULE}},
+    % We wait after the first client so that
+    % the rest of the clients go directly to
+    % ddoc_cache_entry bypassing ddoc_cache_lru
+    Client1 = spawn_monitor(fun() ->
+        ddoc_cache_lru:open(Key)
+    end),
+    meck:wait(ddoc_cache_ev, event, [started, Key], 1000),
+    Clients = [Client1] ++ lists:map(fun(_) ->
+        spawn_monitor(fun() ->
+            ddoc_cache_lru:open(Key)
+        end)
+    end, lists:seq(1, 9)),
+    lists:foreach(fun({Pid, _Ref}) ->
+        ?assert(is_process_alive(Pid))
+    end, Clients),
+    [#entry{pid = Pid}] = ets:tab2list(?CACHE),
+    Opener = element(4, sys:get_state(Pid)),
+    OpenerRef = erlang:monitor(process, Opener),
+    ?assert(is_process_alive(Opener)),
+    Opener ! go,
+    receive {'DOWN', OpenerRef, _, _, _} -> ok end,
+    lists:foreach(fun({_, Ref}) ->
+        receive {'DOWN', Ref, _, _, normal} -> ok end
+    end, Clients).
+
+
+check_capped_size(_) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    lists:foreach(fun(I) ->
+        DbName = list_to_binary(integer_to_list(I)),
+        ddoc_cache:open_custom(DbName, ?MODULE),
+        meck:wait(I, ddoc_cache_ev, event, [started, '_'], 1000),
+        ?assertEqual(I, ets:info(?CACHE, size))
+    end, lists:seq(1, 5)),
+    lists:foreach(fun(I) ->
+        DbName = list_to_binary(integer_to_list(I)),
+        ddoc_cache:open_custom(DbName, ?MODULE),
+        meck:wait(I, ddoc_cache_ev, event, [started, '_'], 1000),
+        ?assertEqual(5, ets:info(?CACHE, size))
+    end, lists:seq(6, 20)).
+
+
+check_full_cache(_) ->
+    ddoc_cache_tutil:clear(),
+    meck:reset(ddoc_cache_ev),
+    lists:foreach(fun(I) ->
+        DbSuffix = list_to_binary(integer_to_list(I)),
+        DbName = <<"pause", DbSuffix/binary>>,
+        spawn(fun() -> ddoc_cache:open_custom(DbName, ?MODULE) end),
+        meck:wait(I, ddoc_cache_ev, event, [started, '_'], 1000),
+        ?assertEqual(I, ets:info(?CACHE, size))
+    end, lists:seq(1, 5)),
+    lists:foreach(fun(I) ->
+        DbSuffix = list_to_binary(integer_to_list(I)),
+        DbName = <<"pause", DbSuffix/binary>>,
+        spawn(fun() -> ddoc_cache:open_custom(DbName, ?MODULE) end),
+        meck:wait(I - 5, ddoc_cache_ev, event, [full, '_'], 1000),
+        ?assertEqual(5, ets:info(?CACHE, size))
+    end, lists:seq(6, 20)).
diff --git a/src/ddoc_cache/test/ddoc_cache_tutil.erl b/src/ddoc_cache/test/ddoc_cache_tutil.erl
index cdd372b..6782b9d 100644
--- a/src/ddoc_cache/test/ddoc_cache_tutil.erl
+++ b/src/ddoc_cache/test/ddoc_cache_tutil.erl
@@ -21,7 +21,7 @@
 
 
 start_couch() ->
-    %purge_modules(),
+    purge_modules(),
     Ctx = test_util:start_couch(?CONFIG_CHAIN, [chttpd, ddoc_cache]),
     TmpDb = ?tempdb(),
     ok = fabric:create_db(TmpDb, [{q, "1"}, {n, "1"}]),

-- 
To stop receiving notification emails like this one, please contact
['"commits@couchdb.apache.org" <commits@couchdb.apache.org>'].

Mime
View raw message