couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bbast...@apache.org
Subject [3/6] couch-index commit: updated refs/heads/8409-view-cache to 5a9f05c
Date Wed, 08 Mar 2017 20:39:27 GMT
Make index cache only have a soft limit


Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-index/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-index/commit/78a65a3f
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-index/tree/78a65a3f
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-index/diff/78a65a3f

Branch: refs/heads/8409-view-cache
Commit: 78a65a3f5eefd5669b096fcbc4806cd9029299fb
Parents: 19a3a49
Author: Benjamin Bastian <benjamin.bastian@gmail.com>
Authored: Thu Mar 2 14:23:24 2017 -0800
Committer: Benjamin Bastian <benjamin.bastian@gmail.com>
Committed: Thu Mar 2 14:23:24 2017 -0800

----------------------------------------------------------------------
 src/couch_index_monitor.erl        |  20 ++---
 src/couch_index_server.erl         | 133 +++++++++++++-------------------
 test/couch_index_monitor_tests.erl |   5 +-
 3 files changed, 61 insertions(+), 97 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-couch-index/blob/78a65a3f/src/couch_index_monitor.erl
----------------------------------------------------------------------
diff --git a/src/couch_index_monitor.erl b/src/couch_index_monitor.erl
index 4cc4bc7..fc2fef1 100644
--- a/src/couch_index_monitor.erl
+++ b/src/couch_index_monitor.erl
@@ -14,7 +14,7 @@
 
 
 -export([
-    spawn_link/2,
+    spawn_link/1,
     close/1,
     set_pid/2,
 
@@ -24,7 +24,7 @@
 ]).
 
 -export([
-    init/2
+    init/1
 ]).
 
 
@@ -34,15 +34,14 @@
 -record(st, {
     name,
     type,
-    is_sys,
     ref,
     client_refs,
     closing
 }).
 
 
-spawn_link(Name, IsSys) ->
-    erlang:spawn_link(?MODULE, init, [Name, IsSys]).
+spawn_link(Name) ->
+    erlang:spawn_link(?MODULE, init, [Name]).
 
 
 close(Monitor) ->
@@ -67,11 +66,11 @@ notify(Monitor, {Client, _}) when is_pid(Client) ->
     notify(Monitor, Client).
 
 
-cancel(Name, {Client, Monitor, IsSys})
+cancel(Name, {Client, Monitor})
         when Client == self(), is_pid(Monitor) ->
     Monitor ! {cancel, self()},
     case (catch ets:update_counter(?BY_COUNTERS, Name, -1)) of
-        0 when not IsSys ->
+        0 ->
             true = ets:insert(?BY_IDLE, {Name}),
             ok;
         _ ->
@@ -79,11 +78,10 @@ cancel(Name, {Client, Monitor, IsSys})
     end.
 
 
-init(Name, IsSys) ->
+init(Name) ->
     {ok, CRefs} = khash:new(),
     loop(#st{
         name = Name,
-        is_sys = IsSys,
         ref = undefined,
         client_refs = CRefs,
         closing = false
@@ -153,10 +151,6 @@ handle_info(Msg, St) ->
 
 maybe_set_idle(St) ->
     case khash:size(St#st.client_refs) of
-        0 when St#st.is_sys ->
-            % System dbs don't go idle so they're
-            % never a candidate to get closed
-            ok;
         0 ->
             % We're now idle
             ets:insert(?BY_IDLE, {St#st.name});

http://git-wip-us.apache.org/repos/asf/couchdb-couch-index/blob/78a65a3f/src/couch_index_server.erl
----------------------------------------------------------------------
diff --git a/src/couch_index_server.erl b/src/couch_index_server.erl
index 0350492..21028a1 100644
--- a/src/couch_index_server.erl
+++ b/src/couch_index_server.erl
@@ -16,7 +16,7 @@
 
 -vsn(2).
 
--export([start_link/0, validate/2, get_index/4, get_index/3, get_index_from_state/3]).
+-export([start_link/0, validate/2, get_index/4, get_index/3, get_index/2]).
 
 -export([init/1, terminate/2, code_change/3]).
 -export([handle_call/3, handle_cast/2, handle_info/2]).
@@ -37,9 +37,8 @@
 
 -record(st, {
     root_dir,
-    max_indexes,
-    open,
-    sys_open
+    soft_max_indexes,
+    open
 }).
 
 start_link() ->
@@ -99,24 +98,20 @@ get_index(Module, Db, DDoc, Fun) when is_binary(DDoc) ->
 get_index(Module, Db, DDoc, Fun) when is_function(Fun, 1) ->
     {ok, InitState} = Module:init(Db, DDoc),
     {ok, FunResp} = Fun(InitState),
-    case get_index_from_state(Module, InitState, couch_db:is_system_db(Db)) of
-        {ok, Pid} ->
-            {ok, Pid, FunResp};
-        {error, all_active} ->
-            {error, all_active}
-    end;
+    {ok, Pid} = get_index(Module, InitState),
+    {ok, Pid, FunResp};
 get_index(Module, Db, DDoc, _Fun) ->
     {ok, InitState} = Module:init(Db, DDoc),
-    get_index_from_state(Module, InitState, couch_db:is_system_db(Db)).
+    get_index(Module, InitState).
 
 
-get_index_from_state(Module, IdxState, SysOwned) ->
+get_index(Module, IdxState) ->
     DbName = Module:get(db_name, IdxState),
     Sig = Module:get(signature, IdxState),
-    Args = {Module, IdxState, DbName, Sig, SysOwned},
+    Args = {Module, IdxState, DbName, Sig},
     case incref({DbName, Sig}) of
         ok ->
-            [{_, {Pid, Monitor, _SysOwned}}] = ets:lookup(?BY_SIG, {DbName, Sig}),
+            [{_, {Pid, Monitor}}] = ets:lookup(?BY_SIG, {DbName, Sig}),
             ok = couch_index_monitor:notify(Monitor),
             {ok, Pid};
         _ ->
@@ -136,8 +131,8 @@ init([]) ->
     RootDir = couch_index_util:root_dir(),
     couch_file:init_delete_dir(RootDir),
     MaxIndexes = list_to_integer(
-        config:get("couchdb", "max_indexes_open", integer_to_list(?MAX_INDEXES_OPEN))),
-    {ok, #st{root_dir=RootDir, max_indexes=MaxIndexes, open=0, sys_open=0}}.
+        config:get("couchdb", "soft_max_indexes_open", integer_to_list(?MAX_INDEXES_OPEN))),
+    {ok, #st{root_dir=RootDir, soft_max_indexes=MaxIndexes, open=0}}.
 
 
 terminate(_Reason, _State) ->
@@ -145,77 +140,60 @@ terminate(_Reason, _State) ->
     lists:map(fun couch_util:shutdown_sync/1, Pids),
     ok.
 
-make_room(State, false) ->
-    case maybe_close_idle(State) of
-        {ok, NewState} ->
-            {ok, NewState};
-        Other ->
-            Other
-    end;
-make_room(State, true) ->
-    {ok, State}.
 
--spec maybe_close_idle(#st{}) -> {ok, #st{}} | {error, all_active}.
-maybe_close_idle(#st{open=Open, max_indexes=Max}=State) when Open < Max ->
+-spec maybe_close_idle(#st{}) -> {ok, #st{}}.
+maybe_close_idle(#st{open=Open, soft_max_indexes=Max}=State) when Open < Max ->
     {ok, State};
 
 maybe_close_idle(State) ->
-    try
-        {ok, close_idle(State)}
-    catch error:all_active ->
-        {error, all_active}
-    end.
-
--spec close_idle(#st{}) -> #st{}.
-close_idle(State) ->
     ets:safe_fixtable(?BY_IDLE, true),
     try
-        close_idle(State, ets:first(?BY_IDLE))
+        {ok, close_idle(State, ets:first(?BY_IDLE))}
     after
         ets:safe_fixtable(?BY_IDLE, false)
     end.
 
 
 -spec close_idle(#st{}, term()) -> #st{}.
-close_idle(_State, '$end_of_table') ->
-    erlang:error(all_active);
+close_idle(State, '$end_of_table') ->
+    State;
 
 close_idle(State, Name) ->
+    true = ets:delete(?BY_IDLE, Name),
     case ets:lookup(?BY_SIG, Name) of
-        [{_, {Pid, _Monitor, SysOwned}}] ->
-            true = ets:delete(?BY_IDLE, Name),
+        [{_, {Pid, _Monitor}}] ->
             couch_index:stop(Pid),
-            closed(State, SysOwned);
+            case closed(State) of
+                NewState when NewState#st.open > State#st.soft_max_indexes ->
+                    close_idle(NewState, ets:next(?BY_IDLE, Name));
+                NewState ->
+                    NewState
+            end;
         [] ->
-            true = ets:delete(?BY_IDLE, Name),
             close_idle(State, ets:next(?BY_IDLE, Name))
     end.
 
 
-handle_call({get_index, {_Mod, _IdxState, DbName, Sig, SysOwned}=Args}, From, State) ->
+handle_call({get_index, {_Mod, _IdxState, DbName, Sig}=Args}, From, State) ->
     case ets:lookup(?BY_SIG, {DbName, Sig}) of
         [] ->
-            case make_room(State, SysOwned) of
-                {ok, NewState} ->
-                    spawn_link(fun() -> new_index(Args) end),
-                    Monitor = couch_index_monitor:spawn_link({DbName, Sig}, SysOwned),
-                    ets:insert(?BY_SIG, {{DbName, Sig}, {[From], Monitor, SysOwned}}),
-                    {noreply, NewState};
-                {error, all_active} ->
-                    {reply, {error, all_active}, State}
-            end;
-        [{_, {Waiters, Monitor, SysOwned}}] when is_list(Waiters) ->
-            ets:insert(?BY_SIG, {{DbName, Sig}, {[From | Waiters], Monitor, SysOwned}}),
+            {ok, NewState} = maybe_close_idle(State),
+            spawn_link(fun() -> new_index(Args) end),
+            Monitor = couch_index_monitor:spawn_link({DbName, Sig}),
+            ets:insert(?BY_SIG, {{DbName, Sig}, {[From], Monitor}}),
+            {noreply, NewState};
+        [{_, {Waiters, Monitor}}] when is_list(Waiters) ->
+            ets:insert(?BY_SIG, {{DbName, Sig}, {[From | Waiters], Monitor}}),
             {noreply, State};
-        [{_, {Pid, Monitor, _SysOwned}}] when is_pid(Pid) ->
+        [{_, {Pid, Monitor}}] when is_pid(Pid) ->
             ok = incref({DbName, Sig}),
             ok = couch_index_monitor:notify(Monitor, From),
             {reply, {ok, Pid}, State}
     end;
 handle_call({async_open, {DbName, DDocId, Sig}, {ok, Pid}}, _From, State) ->
-    [{_, {Waiters, Monitor, SysOwned}}] = ets:lookup(?BY_SIG, {DbName, Sig}),
+    [{_, {Waiters, Monitor}}] = ets:lookup(?BY_SIG, {DbName, Sig}),
     link(Pid),
-    ets:insert(?BY_SIG, {{DbName, Sig}, {Pid, Monitor, SysOwned}}),
+    ets:insert(?BY_SIG, {{DbName, Sig}, {Pid, Monitor}}),
     ets:insert(?BY_PID, {Pid, {DbName, Sig}}),
     ets:insert(?BY_COUNTERS, {{DbName, Sig}, 0}),
     ets:insert(?BY_DB, {DbName, {DDocId, Sig}}),
@@ -225,7 +203,7 @@ handle_call({async_open, {DbName, DDocId, Sig}, {ok, Pid}}, _From, State)
->
         ok = couch_index_monitor:notify(Monitor, Client),
         gen_server:reply(From, {ok, Pid})
     end, Waiters),
-    {reply, ok, opened(State, SysOwned)};
+    {reply, ok, opened(State)};
 handle_call({async_error, {DbName, _DDocId, Sig}, Error}, {FromPid, _}, State) ->
     [{_, {Waiters, Monitor, _SO}}] = ets:lookup(?BY_SIG, {DbName, Sig}),
     [gen_server:reply(From, Error) || From <- Waiters],
@@ -235,12 +213,12 @@ handle_call({async_error, {DbName, _DDocId, Sig}, Error}, {FromPid,
_}, State) -
     true = ets:delete(?BY_IDLE, {DbName, Sig}),
     ok = couch_index_monitor:close(Monitor),
     {reply, ok, State};
-handle_call({set_max_indexes_open, Max}, _From, State) ->
-    {reply, ok, State#st{max_indexes=Max}};
+handle_call({set_soft_max_indexes_open, Max}, _From, State) ->
+    {reply, ok, State#st{soft_max_indexes=Max}};
 handle_call({reset_indexes, DbName}, _From, State) ->
     {reply, ok, reset_indexes(DbName, State)};
 handle_call(open_index_count, _From, State) ->
-    {reply, {State#st.open, State#st.sys_open}, State};
+    {reply, State#st.open, State};
 handle_call(get_server, _From, State) ->
     {reply, State, State}.
 
@@ -251,11 +229,10 @@ handle_cast({reset_indexes, DbName}, State) ->
 handle_info({'EXIT', Pid, Reason}, Server) ->
     case ets:lookup(?BY_PID, Pid) of
         [{_, {DbName, Sig}}] ->
-            [{_, {_W, _M, SysOwned}}] = ets:lookup(?BY_SIG, {DbName, Sig}),
             [{DbName, {DDocId, Sig}}] =
                 ets:match_object(?BY_DB, {DbName, {'$1', Sig}}),
             rem_from_ets(DbName, Sig, DDocId, Pid),
-            {noreply, closed(Server, SysOwned)};
+            {noreply, closed(Server)};
         [] when Reason /= normal ->
             exit(Reason);
         _Else ->
@@ -283,8 +260,8 @@ handle_config_change("couchdb", "index_dir", _, _, _) ->
 handle_config_change("couchdb", "view_index_dir", _, _, _) ->
     exit(whereis(couch_index_server), config_change),
     remove_handler;
-handle_config_change("couchdb", "max_indexes_open", Max, _, _) when is_list(Max) ->
-    {ok, gen_server:call(?MODULE, {set_max_indexes_open, list_to_integer(Max)})};
+handle_config_change("couchdb", "soft_max_indexes_open", Max, _, _) when is_list(Max) ->
+    {ok, gen_server:call(?MODULE, {set_soft_max_indexes_open, list_to_integer(Max)})};
 handle_config_change(_, _, _, _, _) ->
     {ok, nil}.
 
@@ -295,7 +272,7 @@ handle_config_terminate(_Server, _Reason, _State) ->
     {ok, couch_index_util:root_dir()}.
 
 
-new_index({Mod, IdxState, DbName, Sig, _SysOwned}) ->
+new_index({Mod, IdxState, DbName, Sig}) ->
     DDocId = Mod:get(idx_name, IdxState),
     case couch_index:start_link({Mod, IdxState}) of
         {ok, Pid} ->
@@ -312,13 +289,13 @@ reset_indexes(DbName, State) ->
     #st{root_dir=Root} = State,
     % shutdown all the updaters and clear the files, the db got changed
     Fun = fun({_, {DDocId, Sig}}, StateAcc) ->
-        [{_, {Pid, Monitor, SysOwned}}] = ets:lookup(?BY_SIG, {DbName, Sig}),
+        [{_, {Pid, Monitor}}] = ets:lookup(?BY_SIG, {DbName, Sig}),
         couch_index_monitor:close(Monitor),
         MRef = erlang:monitor(process, Pid),
         gen_server:cast(Pid, delete),
         receive {'DOWN', MRef, _, _, _} -> ok end,
         rem_from_ets(DbName, Sig, DDocId, Pid),
-        closed(StateAcc, SysOwned)
+        closed(StateAcc)
     end,
     NewState = lists:foldl(Fun, State, ets:lookup(?BY_DB, DbName)),
     Path = couch_index_util:index_dir("", DbName),
@@ -343,7 +320,7 @@ handle_db_event(DbName, deleted, St) ->
 handle_db_event(DbName, {ddoc_updated, DDocId}, St) ->
     lists:foreach(fun({_DbName, {_DDocId, Sig}}) ->
         case ets:lookup(?BY_SIG, {DbName, Sig}) of
-            [{_, {IndexPid, _Monitor, _SysOwned}}] ->
+            [{_, {IndexPid, _Monitor}}] ->
                 (catch gen_server:cast(IndexPid, ddoc_updated));
             [] ->
                 ok
@@ -354,20 +331,14 @@ handle_db_event(_DbName, _Event, St) ->
     {ok, St}.
 
 
--spec opened(#st{}, boolean()) -> #st{}.
-opened(State, IsSysOwned) ->
-    case IsSysOwned of
-        true -> State#st{sys_open=State#st.sys_open + 1};
-        false -> State#st{open=State#st.open + 1}
-    end.
+-spec opened(#st{}) -> #st{}.
+opened(State) ->
+    State#st{open=State#st.open + 1}.
 
 
--spec closed(#st{}, boolean()) -> #st{}.
-closed(State, IsSysOwned) ->
-    case IsSysOwned of
-        true -> State#st{sys_open=State#st.sys_open - 1};
-        false -> State#st{open=State#st.open - 1}
-    end.
+-spec closed(#st{}) -> #st{}.
+closed(State) ->
+    State#st{open=State#st.open - 1}.
 
 
 incref(Name) ->

http://git-wip-us.apache.org/repos/asf/couchdb-couch-index/blob/78a65a3f/test/couch_index_monitor_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_index_monitor_tests.erl b/test/couch_index_monitor_tests.erl
index d4fd2c9..9e0d8e0 100644
--- a/test/couch_index_monitor_tests.erl
+++ b/test/couch_index_monitor_tests.erl
@@ -60,10 +60,9 @@ test_basic({Db, DDoc}) ->
         {ok, IdxState} = couch_mrview_index:init(Db, DDoc),
         DbName = couch_mrview_index:get(db_name, IdxState),
         Sig = couch_mrview_index:get(signature, IdxState),
-        [{_, {Pid, Monitor, SysOwned}}] = ets:lookup(?BY_SIG, {DbName, Sig}),
+        [{_, {Pid, Monitor}}] = ets:lookup(?BY_SIG, {DbName, Sig}),
         ?assert(is_pid(Pid)),
         ?assert(is_pid(Monitor)),
-        ?assertNot(SysOwned),
         ?assertEqual(1, get_count({DbName, Sig})),
         Acq1 = spawn_monitor_acquirer(DbName, DDoc#doc.id),
         ?assertEqual(2, get_count({DbName, Sig})),
@@ -73,7 +72,7 @@ test_basic({Db, DDoc}) ->
         ?assertEqual(2, get_count({DbName, Sig})),
         wait_down(Acq1),
         ?assertEqual(1, get_count({DbName, Sig})),
-        couch_index_monitor:cancel({DbName, Sig}, {self(), Monitor, SysOwned}),
+        couch_index_monitor:cancel({DbName, Sig}, {self(), Monitor}),
         ?assertEqual(0, get_count({DbName, Sig})),
         ?assertEqual(1, length(ets:lookup(?BY_IDLE, {DbName, Sig}))),
         ok


Mime
View raw message