Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 7FBD2200CB6 for ; Thu, 29 Jun 2017 23:11:33 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 7E572160BC6; Thu, 29 Jun 2017 21:11:33 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 76B57160BED for ; Thu, 29 Jun 2017 23:11:32 +0200 (CEST) Received: (qmail 18478 invoked by uid 500); 29 Jun 2017 21:11:22 -0000 Mailing-List: contact commits-help@couchdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@couchdb.apache.org Delivered-To: mailing list commits@couchdb.apache.org Received: (qmail 18469 invoked by uid 99); 29 Jun 2017 21:11:22 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 29 Jun 2017 21:11:22 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 98EF187545; Thu, 29 Jun 2017 21:11:19 +0000 (UTC) Date: Thu, 29 Jun 2017 21:11:19 +0000 To: "commits@couchdb.apache.org" Subject: [couchdb] branch optimize-ddoc-cache updated: SQUAHS: Test suite updates MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <149877067974.22735.8266453202411175977@gitbox.apache.org> From: davisp@apache.org Reply-To: "commits@couchdb.apache.org" X-Git-Host: gitbox.apache.org X-Git-Repo: couchdb X-Git-Refname: refs/heads/optimize-ddoc-cache X-Git-Reftype: branch X-Git-Oldrev: 629c2cf50dcf405ce80dc66b406c6af1c61b8172 X-Git-Newrev: d022f9af725386289aad1301677c087207861f3b X-Git-Rev: d022f9af725386289aad1301677c087207861f3b X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated archived-at: Thu, 29 Jun 2017 21:11:33 -0000 This is an automated email from the ASF dual-hosted git repository. davisp pushed a commit to branch optimize-ddoc-cache in repository https://gitbox.apache.org/repos/asf/couchdb.git The following commit(s) were added to refs/heads/optimize-ddoc-cache by this push: new d022f9a SQUAHS: Test suite updates d022f9a is described below commit d022f9af725386289aad1301677c087207861f3b Author: Paul J. Davis AuthorDate: Thu Jun 29 16:10:58 2017 -0500 SQUAHS: Test suite updates --- src/ddoc_cache/src/ddoc_cache_entry.erl | 9 +- src/ddoc_cache/src/ddoc_cache_lru.erl | 30 +++-- src/ddoc_cache/test/ddoc_cache_coverage_test.erl | 6 +- src/ddoc_cache/test/ddoc_cache_entry_test.erl | 120 ++++++++++++++++++ src/ddoc_cache/test/ddoc_cache_eviction_test.erl | 2 + src/ddoc_cache/test/ddoc_cache_lru_test.erl | 153 +++++++++++++++++++++++ src/ddoc_cache/test/ddoc_cache_tutil.erl | 2 +- 7 files changed, 300 insertions(+), 22 deletions(-) diff --git a/src/ddoc_cache/src/ddoc_cache_entry.erl b/src/ddoc_cache/src/ddoc_cache_entry.erl index 4689390..4dee2a1 100644 --- a/src/ddoc_cache/src/ddoc_cache_entry.erl +++ b/src/ddoc_cache/src/ddoc_cache_entry.erl @@ -98,17 +98,13 @@ refresh(Pid) -> init(Key) -> - Entry = #entry{ - key = Key, - val = undefined, - pid = self() - }, - true = ets:insert(?CACHE, Entry), + true = ets:update_element(?CACHE, Key, {#entry.pid, self()}), St = #st{ key = Key, opener = spawn_opener(Key), waiters = [] }, + ?EVENT(started, Key), gen_server:enter_loop(?MODULE, [], St). @@ -281,4 +277,3 @@ drain_accessed() -> respond(Waiters, Resp) -> [gen_server:reply(W, Resp) || W <- Waiters]. - diff --git a/src/ddoc_cache/src/ddoc_cache_lru.erl b/src/ddoc_cache/src/ddoc_cache_lru.erl index 5988f03..14440f2 100644 --- a/src/ddoc_cache/src/ddoc_cache_lru.erl +++ b/src/ddoc_cache/src/ddoc_cache_lru.erl @@ -69,8 +69,8 @@ open(Key) -> ddoc_cache_entry:accessed(Pid), {ok, Val} catch _:_ -> - couch_stats:increment_counter([ddoc_cache, recovery]), - ddoc_cache_entry:recover(Key) + couch_stats:increment_counter([ddoc_cache, recovery]), + ddoc_cache_entry:recover(Key) end. @@ -107,15 +107,23 @@ handle_call({start, Key}, _From, St) -> dbs = Dbs, size = CurSize } = St, - MaxSize = max(0, config:get_integer("ddoc_cache", "max_size", 500)), - case trim(St, CurSize, MaxSize) of - {ok, N} -> - {ok, Pid} = ddoc_cache_entry:start_link(Key), - ok = khash:put(Pids, Pid, Key), - store_key(Dbs, Key, Pid), - {reply, {ok, Pid}, St#st{size = CurSize - N + 1}}; - full -> - {reply, full, St} + case ets:lookup(?CACHE, Key) of + [] -> + MaxSize = config:get_integer("ddoc_cache", "max_size", 1000), + case trim(St, CurSize, max(0, MaxSize)) of + {ok, N} -> + true = ets:insert_new(?CACHE, #entry{key = Key}), + {ok, Pid} = ddoc_cache_entry:start_link(Key), + true = ets:update_element(?CACHE, Key, {#entry.pid, Pid}), + ok = khash:put(Pids, Pid, Key), + store_key(Dbs, Key, Pid), + {reply, {ok, Pid}, St#st{size = CurSize - N + 1}}; + full -> + ?EVENT(full, Key), + {reply, full, St} + end; + [#entry{pid = Pid}] -> + {reply, {ok, Pid}, St} end; handle_call(Msg, _From, St) -> diff --git a/src/ddoc_cache/test/ddoc_cache_coverage_test.erl b/src/ddoc_cache/test/ddoc_cache_coverage_test.erl index 91182ca..395f560 100644 --- a/src/ddoc_cache/test/ddoc_cache_coverage_test.erl +++ b/src/ddoc_cache/test/ddoc_cache_coverage_test.erl @@ -33,7 +33,7 @@ coverage_test_() -> restart_lru() -> send_bad_messages(ddoc_cache_lru), - ?assertEqual(ok, ddoc_cache_lru:terminate(bang, {st, a, b, c})), + ?assertEqual(ok, ddoc_cache_lru:terminate(bang, {st, a, b, c, d})), ?assertEqual({ok, foo}, ddoc_cache_lru:code_change(1, foo, [])). @@ -47,7 +47,7 @@ restart_evictor() -> meck:new(ddoc_cache_ev, [passthrough]), try State = sys:get_state(ddoc_cache_lru), - Evictor = element(4, State), + Evictor = element(5, State), Ref = erlang:monitor(process, Evictor), exit(Evictor, shutdown), receive @@ -57,7 +57,7 @@ restart_evictor() -> end, meck:wait(ddoc_cache_ev, event, [evictor_died, '_'], 1000), NewState = sys:get_state(ddoc_cache_lru), - NewEvictor = element(4, NewState), + NewEvictor = element(5, NewState), ?assertNotEqual(Evictor, NewEvictor) after meck:unload() diff --git a/src/ddoc_cache/test/ddoc_cache_entry_test.erl b/src/ddoc_cache/test/ddoc_cache_entry_test.erl new file mode 100644 index 0000000..62afc72 --- /dev/null +++ b/src/ddoc_cache/test/ddoc_cache_entry_test.erl @@ -0,0 +1,120 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(ddoc_cache_entry_test). + + +-export([ + recover/1 +]). + + +-include_lib("couch/include/couch_db.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include("ddoc_cache_test.hrl"). + + +recover(<<"foo">>) -> + timer:sleep(30000); + +recover(DbName) -> + {ok, {DbName, such_custom}}. + + +start_couch() -> + Ctx = ddoc_cache_tutil:start_couch(), + meck:new(ddoc_cache_ev, [passthrough]), + Ctx. + + +stop_couch(Ctx) -> + meck:unload(), + ddoc_cache_tutil:stop_couch(Ctx). + + +check_entry_test_() -> + { + setup, + fun start_couch/0, + fun stop_couch/1, + {with, [ + fun cancel_and_replace_opener/1, + fun condenses_access_messages/1, + fun kill_opener_on_terminate/1, + fun open_dead_entry/1, + fun handles_bad_messages/1, + fun handles_code_change/1 + ]} + }. + + +cancel_and_replace_opener(_) -> + Key = {ddoc_cache_entry_custom, {<<"foo">>, ?MODULE}}, + true = ets:insert_new(?CACHE, #entry{key = Key}), + {ok, Entry} = ddoc_cache_entry:start_link(Key), + Opener1 = element(4, sys:get_state(Entry)), + Ref1 = erlang:monitor(process, Opener1), + gen_server:cast(Entry, refresh), + receive {'DOWN', Ref1, _, _, _} -> ok end, + Opener2 = element(4, sys:get_state(Entry)), + ?assert(Opener2 /= Opener1), + ?assert(is_process_alive(Opener2)), + % Clean up after ourselves + unlink(Entry), + ddoc_cache_entry:shutdown(Entry). + + +condenses_access_messages({DbName, _}) -> + meck:reset(ddoc_cache_ev), + Key = {ddoc_cache_entry_custom, {DbName, ?MODULE}}, + true = ets:insert(?CACHE, #entry{key = Key}), + {ok, Entry} = ddoc_cache_entry:start_link(Key), + erlang:suspend_process(Entry), + lists:foreach(fun(_) -> + gen_server:cast(Entry, accessed) + end, lists:seq(1, 100)), + erlang:resume_process(Entry), + meck:wait(1, ddoc_cache_ev, event, [accessed, Key], 1000), + ?assertError( + timeout, + meck:wait(2, ddoc_cache_ev, event, [accessed, Key], 100) + ), + unlink(Entry), + ddoc_cache_entry:shutdown(Entry). + + +kill_opener_on_terminate(_) -> + Pid = spawn(fun() -> receive _ -> ok end end), + ?assert(is_process_alive(Pid)), + St = {st, key, val, Pid, waiters, ts}, + ?assertEqual(ok, ddoc_cache_entry:terminate(normal, St)), + ?assert(not is_process_alive(Pid)). + + +open_dead_entry({DbName, _}) -> + Pid = spawn(fun() -> ok end), + Key = {ddoc_cache_entry_custom, {DbName, ?MODULE}}, + ?assertEqual(recover(DbName), ddoc_cache_entry:open(Pid, Key)). + + +handles_bad_messages(_) -> + CallExpect = {stop, {bad_call, foo}, {bad_call, foo}, baz}, + CastExpect = {stop, {bad_cast, foo}, bar}, + InfoExpect = {stop, {bad_info, foo}, bar}, + ?assertEqual(CallExpect, ddoc_cache_entry:handle_call(foo, bar, baz)), + ?assertEqual(CastExpect, ddoc_cache_entry:handle_cast(foo, bar)), + ?assertEqual(InfoExpect, ddoc_cache_entry:handle_info(foo, bar)). + + +handles_code_change(_) -> + CCExpect = {ok, bar}, + ?assertEqual(CCExpect, ddoc_cache_entry:code_change(foo, bar, baz)). diff --git a/src/ddoc_cache/test/ddoc_cache_eviction_test.erl b/src/ddoc_cache/test/ddoc_cache_eviction_test.erl index 62ed0e8..0b9f57b 100644 --- a/src/ddoc_cache/test/ddoc_cache_eviction_test.erl +++ b/src/ddoc_cache/test/ddoc_cache_eviction_test.erl @@ -63,6 +63,7 @@ evict_all({DbName, _}) -> ?assertEqual(4, ets:info(?CACHE, size)), {ok, _} = ddoc_cache_lru:handle_db_event(ShardName, deleted, foo), meck:wait(ddoc_cache_ev, event, [evicted, DbName], 1000), + meck:wait(4, ddoc_cache_ev, event, [removed, '_'], 1000), ?assertEqual(0, ets:info(?CACHE, size)). @@ -88,4 +89,5 @@ check_upgrade_clause({DbName, _}) -> ?assertEqual(1, ets:info(?CACHE, size)), gen_server:cast(ddoc_cache_opener, {do_evict, DbName}), meck:wait(ddoc_cache_ev, event, [evicted, DbName], 1000), + meck:wait(ddoc_cache_ev, event, [removed, '_'], 1000), ?assertEqual(0, ets:info(?CACHE, size)). diff --git a/src/ddoc_cache/test/ddoc_cache_lru_test.erl b/src/ddoc_cache/test/ddoc_cache_lru_test.erl new file mode 100644 index 0000000..c3058ff --- /dev/null +++ b/src/ddoc_cache/test/ddoc_cache_lru_test.erl @@ -0,0 +1,153 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(ddoc_cache_lru_test). + + +-export([ + recover/1 +]). + + +-include_lib("couch/include/couch_db.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include("ddoc_cache_test.hrl"). + + +recover(<<"pause", _/binary>>) -> + receive go -> ok end, + {ok, paused}; + +recover(DbName) -> + {ok, DbName}. + + +start_couch() -> + Ctx = ddoc_cache_tutil:start_couch(), + config:set("ddoc_cache", "max_size", "5", false), + meck:new(ddoc_cache_ev, [passthrough]), + Ctx. + + +stop_couch(Ctx) -> + meck:unload(), + ddoc_cache_tutil:stop_couch(Ctx). + + +check_not_started_test() -> + Key = {ddoc_cache_entry_custom, {<<"dbname">>, ?MODULE}}, + ?assertEqual({ok, <<"dbname">>}, ddoc_cache_lru:open(Key)). + + +check_lru_test_() -> + { + setup, + fun start_couch/0, + fun stop_couch/1, + {with, [ + fun check_multi_start/1, + fun check_multi_open/1, + fun check_capped_size/1, + fun check_full_cache/1 + ]} + }. + + +check_multi_start(_) -> + ddoc_cache_tutil:clear(), + meck:reset(ddoc_cache_ev), + Key = {ddoc_cache_entry_custom, {<<"pause">>, ?MODULE}}, + % These will all get sent through ddoc_cache_lru + Clients = lists:map(fun(_) -> + spawn_monitor(fun() -> + ddoc_cache_lru:open(Key) + end) + end, lists:seq(1, 10)), + meck:wait(ddoc_cache_ev, event, [started, Key], 1000), + lists:foreach(fun({Pid, _Ref}) -> + ?assert(is_process_alive(Pid)) + end, Clients), + [#entry{pid = Pid}] = ets:tab2list(?CACHE), + Opener = element(4, sys:get_state(Pid)), + OpenerRef = erlang:monitor(process, Opener), + ?assert(is_process_alive(Opener)), + Opener ! go, + receive {'DOWN', OpenerRef, _, _, _} -> ok end, + lists:foreach(fun({_, Ref}) -> + receive {'DOWN', Ref, _, _, normal} -> ok end + end, Clients). + + +check_multi_open(_) -> + ddoc_cache_tutil:clear(), + meck:reset(ddoc_cache_ev), + Key = {ddoc_cache_entry_custom, {<<"pause">>, ?MODULE}}, + % We wait after the first client so that + % the rest of the clients go directly to + % ddoc_cache_entry bypassing ddoc_cache_lru + Client1 = spawn_monitor(fun() -> + ddoc_cache_lru:open(Key) + end), + meck:wait(ddoc_cache_ev, event, [started, Key], 1000), + Clients = [Client1] ++ lists:map(fun(_) -> + spawn_monitor(fun() -> + ddoc_cache_lru:open(Key) + end) + end, lists:seq(1, 9)), + lists:foreach(fun({Pid, _Ref}) -> + ?assert(is_process_alive(Pid)) + end, Clients), + [#entry{pid = Pid}] = ets:tab2list(?CACHE), + Opener = element(4, sys:get_state(Pid)), + OpenerRef = erlang:monitor(process, Opener), + ?assert(is_process_alive(Opener)), + Opener ! go, + receive {'DOWN', OpenerRef, _, _, _} -> ok end, + lists:foreach(fun({_, Ref}) -> + receive {'DOWN', Ref, _, _, normal} -> ok end + end, Clients). + + +check_capped_size(_) -> + ddoc_cache_tutil:clear(), + meck:reset(ddoc_cache_ev), + lists:foreach(fun(I) -> + DbName = list_to_binary(integer_to_list(I)), + ddoc_cache:open_custom(DbName, ?MODULE), + meck:wait(I, ddoc_cache_ev, event, [started, '_'], 1000), + ?assertEqual(I, ets:info(?CACHE, size)) + end, lists:seq(1, 5)), + lists:foreach(fun(I) -> + DbName = list_to_binary(integer_to_list(I)), + ddoc_cache:open_custom(DbName, ?MODULE), + meck:wait(I, ddoc_cache_ev, event, [started, '_'], 1000), + ?assertEqual(5, ets:info(?CACHE, size)) + end, lists:seq(6, 20)). + + +check_full_cache(_) -> + ddoc_cache_tutil:clear(), + meck:reset(ddoc_cache_ev), + lists:foreach(fun(I) -> + DbSuffix = list_to_binary(integer_to_list(I)), + DbName = <<"pause", DbSuffix/binary>>, + spawn(fun() -> ddoc_cache:open_custom(DbName, ?MODULE) end), + meck:wait(I, ddoc_cache_ev, event, [started, '_'], 1000), + ?assertEqual(I, ets:info(?CACHE, size)) + end, lists:seq(1, 5)), + lists:foreach(fun(I) -> + DbSuffix = list_to_binary(integer_to_list(I)), + DbName = <<"pause", DbSuffix/binary>>, + spawn(fun() -> ddoc_cache:open_custom(DbName, ?MODULE) end), + meck:wait(I - 5, ddoc_cache_ev, event, [full, '_'], 1000), + ?assertEqual(5, ets:info(?CACHE, size)) + end, lists:seq(6, 20)). diff --git a/src/ddoc_cache/test/ddoc_cache_tutil.erl b/src/ddoc_cache/test/ddoc_cache_tutil.erl index cdd372b..6782b9d 100644 --- a/src/ddoc_cache/test/ddoc_cache_tutil.erl +++ b/src/ddoc_cache/test/ddoc_cache_tutil.erl @@ -21,7 +21,7 @@ start_couch() -> - %purge_modules(), + purge_modules(), Ctx = test_util:start_couch(?CONFIG_CHAIN, [chttpd, ddoc_cache]), TmpDb = ?tempdb(), ok = fabric:create_db(TmpDb, [{q, "1"}, {n, "1"}]), -- To stop receiving notification emails like this one, please contact ['"commits@couchdb.apache.org" '].