Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 9440D200C5B for ; Tue, 4 Apr 2017 23:37:46 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 92EC4160BA1; Tue, 4 Apr 2017 21:37:46 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 35BB0160BA9 for ; Tue, 4 Apr 2017 23:37:45 +0200 (CEST) Received: (qmail 52452 invoked by uid 500); 4 Apr 2017 21:37:43 -0000 Mailing-List: contact commits-help@couchdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@couchdb.apache.org Delivered-To: mailing list commits@couchdb.apache.org Received: (qmail 52007 invoked by uid 99); 4 Apr 2017 21:37:43 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 04 Apr 2017 21:37:43 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8E01FE965C; Tue, 4 Apr 2017 21:37:43 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: davisp@apache.org To: commits@couchdb.apache.org Date: Tue, 04 Apr 2017 21:37:57 -0000 Message-Id: <16dcfee1421342e2aec22c3e63ff554e@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [15/27] couch commit: updated refs/heads/COUCHDB-3287-mixed-db-records to 5f6ff5a archived-at: Tue, 04 Apr 2017 21:37:46 -0000 Add sys_dbs to the LRU COUCHDB-3325 Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/92c25a98 Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/92c25a98 Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/92c25a98 Branch: refs/heads/COUCHDB-3287-mixed-db-records Commit: 92c25a98666e59ca497fa5732a81e771ff52e07d Parents: 5685f17 Author: Robert Newson Authored: Tue Mar 14 17:40:21 2017 +0000 Committer: Robert Newson Committed: Tue Mar 14 23:59:11 2017 +0000 ---------------------------------------------------------------------- src/couch_db_updater.erl | 7 +--- src/couch_file.erl | 22 +++--------- src/couch_lru.erl | 5 +-- src/couch_server.erl | 75 ++++++++++++++------------------------- test/couchdb_views_tests.erl | 2 -- 5 files changed, 35 insertions(+), 76 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/92c25a98/src/couch_db_updater.erl ---------------------------------------------------------------------- diff --git a/src/couch_db_updater.erl b/src/couch_db_updater.erl index 7872635..270fffe 100644 --- a/src/couch_db_updater.erl +++ b/src/couch_db_updater.erl @@ -61,12 +61,7 @@ init({DbName, Filepath, Fd, Options}) -> end end, Db = init_db(DbName, Filepath, Fd, Header, Options), - case lists:member(sys_db, Options) of - false -> - couch_stats_process_tracker:track([couchdb, open_databases]); - true -> - ok - end, + couch_stats_process_tracker:track([couchdb, open_databases]), % we don't load validation funs here because the fabric query is liable to % race conditions. Instead see couch_db:validate_doc_update, which loads % them lazily http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/92c25a98/src/couch_file.erl ---------------------------------------------------------------------- diff --git a/src/couch_file.erl b/src/couch_file.erl index d40c525..d7b176e 100644 --- a/src/couch_file.erl +++ b/src/couch_file.erl @@ -362,7 +362,7 @@ init({Filepath, Options, ReturnPid, Ref}) -> {ok, 0} = file:position(Fd, 0), ok = file:truncate(Fd), ok = file:sync(Fd), - maybe_track_open_os_files(Options), + couch_stats_process_tracker:track([couchdb, open_os_files]), erlang:send_after(?INITIAL_WAIT, self(), maybe_close), {ok, #file{fd=Fd, is_sys=IsSys, pread_limit=Limit}}; false -> @@ -370,7 +370,7 @@ init({Filepath, Options, ReturnPid, Ref}) -> init_status_error(ReturnPid, Ref, {error, eexist}) end; false -> - maybe_track_open_os_files(Options), + couch_stats_process_tracker:track([couchdb, open_os_files]), erlang:send_after(?INITIAL_WAIT, self(), maybe_close), {ok, #file{fd=Fd, is_sys=IsSys, pread_limit=Limit}} end; @@ -385,7 +385,7 @@ init({Filepath, Options, ReturnPid, Ref}) -> %% Save Fd in process dictionary for debugging purposes put(couch_file_fd, {Fd, Filepath}), ok = file:close(Fd_Read), - maybe_track_open_os_files(Options), + couch_stats_process_tracker:track([couchdb, open_os_files]), {ok, Eof} = file:position(Fd, eof), erlang:send_after(?INITIAL_WAIT, self(), maybe_close), {ok, #file{fd=Fd, eof=Eof, is_sys=IsSys, pread_limit=Limit}}; @@ -402,14 +402,6 @@ file_open_options(Options) -> [append] end. -maybe_track_open_os_files(Options) -> - case not lists:member(sys_db, Options) of - true -> - couch_stats_process_tracker:track([couchdb, open_os_files]); - false -> - ok - end. - terminate(_Reason, #file{fd = nil}) -> ok; terminate(_Reason, #file{fd = Fd}) -> @@ -676,13 +668,7 @@ split_iolist([Byte | Rest], SplitAt, BeginAcc) when is_integer(Byte) -> split_iolist(Rest, SplitAt - 1, [Byte | BeginAcc]). -% System dbs aren't monitored by couch_stats_process_tracker -is_idle(#file{is_sys=true}) -> - case process_info(self(), monitored_by) of - {monitored_by, []} -> true; - _ -> false - end; -is_idle(#file{is_sys=false}) -> +is_idle(#file{}) -> Tracker = whereis(couch_stats_process_tracker), case process_info(self(), monitored_by) of {monitored_by, []} -> true; http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/92c25a98/src/couch_lru.erl ---------------------------------------------------------------------- diff --git a/src/couch_lru.erl b/src/couch_lru.erl index d58eb69..84d7bee 100644 --- a/src/couch_lru.erl +++ b/src/couch_lru.erl @@ -34,13 +34,14 @@ update(DbName, {Tree0, Dict0}) -> {Tree0, Dict0} end. +%% Attempt to close the oldest idle database. close({Tree, _} = Cache) -> close_int(gb_trees:next(gb_trees:iterator(Tree)), Cache). %% internals close_int(none, _) -> - erlang:error(all_dbs_active); + false; close_int({Lru, DbName, Iter}, {Tree, Dict} = Cache) -> case ets:update_element(couch_dbs, DbName, {#db.fd_monitor, locked}) of true -> @@ -49,7 +50,7 @@ close_int({Lru, DbName, Iter}, {Tree, Dict} = Cache) -> true = ets:delete(couch_dbs, DbName), true = ets:delete(couch_dbs_pid_to_name, Pid), exit(Pid, kill), - {gb_trees:delete(Lru, Tree), dict:erase(DbName, Dict)}; + {true, {gb_trees:delete(Lru, Tree), dict:erase(DbName, Dict)}}; false -> true = ets:update_element(couch_dbs, DbName, {#db.fd_monitor, nil}), couch_stats:increment_counter([couchdb, couch_server, lru_skip]), http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/92c25a98/src/couch_server.erl ---------------------------------------------------------------------- diff --git a/src/couch_server.erl b/src/couch_server.erl index 893b957..1272e7a 100644 --- a/src/couch_server.erl +++ b/src/couch_server.erl @@ -73,8 +73,8 @@ sup_start_link() -> open(DbName, Options0) -> Ctx = couch_util:get_value(user_ctx, Options0, #user_ctx{}), case ets:lookup(couch_dbs, DbName) of - [#db{fd=Fd, fd_monitor=Lock, options=Options} = Db] when Lock =/= locked -> - update_lru(DbName, Options), + [#db{fd=Fd, fd_monitor=Lock} = Db] when Lock =/= locked -> + update_lru(DbName), {ok, Db#db{user_ctx=Ctx, fd_monitor=erlang:monitor(process,Fd)}}; _ -> Options = maybe_add_sys_db_callbacks(DbName, Options0), @@ -91,11 +91,8 @@ open(DbName, Options0) -> end end. -update_lru(DbName, Options) -> - case lists:member(sys_db, Options) of - false -> gen_server:cast(couch_server, {update_lru, DbName}); - true -> ok - end. +update_lru(DbName) -> + gen_server:cast(couch_server, {update_lru, DbName}). close_lru() -> gen_server:call(couch_server, close_lru). @@ -271,20 +268,15 @@ all_databases(Fun, Acc0) -> {ok, FinalAcc}. -make_room(Server, Options) -> - case lists:member(sys_db, Options) of - false -> maybe_close_lru_db(Server); - true -> {ok, Server} - end. - maybe_close_lru_db(#server{dbs_open=NumOpen, max_dbs_open=MaxOpen}=Server) when NumOpen < MaxOpen -> {ok, Server}; maybe_close_lru_db(#server{lru=Lru}=Server) -> - try - {ok, db_closed(Server#server{lru = couch_lru:close(Lru)}, [])} - catch error:all_dbs_active -> - {error, all_dbs_active} + case couch_lru:close(Lru) of + false -> + {ok, Server}; + {true, NewLru} -> + maybe_close_lru_db(db_closed(Server#server{lru = NewLru})) end. open_async(Server, From, DbName, Filepath, Options) -> @@ -316,13 +308,14 @@ open_async(Server, From, DbName, Filepath, Options) -> options = Options }), true = ets:insert(couch_dbs_pid_to_name, {Opener, DbName}), - db_opened(Server, Options). + db_opened(Server). handle_call(close_lru, _From, #server{lru=Lru} = Server) -> - try - {reply, ok, db_closed(Server#server{lru = couch_lru:close(Lru)}, [])} - catch error:all_dbs_active -> - {reply, {error, all_dbs_active}, Server} + case couch_lru:close(Lru) of + false -> + {reply, ok, Server}; + {true, NewLru} -> + {reply, ok, db_closed(Server#server{lru = NewLru})} end; handle_call(open_dbs_count, _From, Server) -> {reply, Server#server.dbs_open, Server}; @@ -355,12 +348,7 @@ handle_call({open_result, T0, DbName, {ok, Db}}, {FromPid, _Tag}, Server) -> end, true = ets:insert(couch_dbs, Db), true = ets:insert(couch_dbs_pid_to_name, {Db#db.main_pid, DbName}), - Lru = case couch_db:is_system_db(Db) of - false -> - couch_lru:insert(DbName, Server#server.lru); - true -> - Server#server.lru - end, + Lru = couch_lru:insert(DbName, Server#server.lru), {reply, ok, Server#server{lru = Lru}} end; handle_call({open_result, T0, DbName, {error, eexist}}, From, Server) -> @@ -382,7 +370,7 @@ handle_call({open_result, _T0, DbName, Error}, {FromPid, _Tag}, Server) -> _ -> Server end, - {reply, ok, db_closed(NewServer, Db#db.options)} + {reply, ok, db_closed(NewServer)} end; handle_call({open, DbName, Options}, From, Server) -> case ets:lookup(couch_dbs, DbName) of @@ -390,7 +378,7 @@ handle_call({open, DbName, Options}, From, Server) -> DbNameList = binary_to_list(DbName), case check_dbname(Server, DbNameList) of ok -> - case make_room(Server, Options) of + case maybe_close_lru_db(Server) of {ok, Server2} -> Filepath = get_full_filename(Server, DbNameList), {noreply, open_async(Server2, From, DbName, Filepath, Options)}; @@ -418,7 +406,7 @@ handle_call({create, DbName, Options}, From, Server) -> ok -> case ets:lookup(couch_dbs, DbName) of [] -> - case make_room(Server, Options) of + case maybe_close_lru_db(Server) of {ok, Server2} -> {noreply, open_async(Server2, From, DbName, Filepath, [create | Options])}; @@ -454,12 +442,12 @@ handle_call({delete, DbName, Options}, _From, Server) -> true = ets:delete(couch_dbs_pid_to_name, Pid), exit(Pid, kill), [gen_server:reply(F, not_found) || F <- Froms], - db_closed(Server, Db#db.options); + db_closed(Server); [#db{main_pid=Pid} = Db] -> true = ets:delete(couch_dbs, DbName), true = ets:delete(couch_dbs_pid_to_name, Pid), exit(Pid, kill), - db_closed(Server, Db#db.options) + db_closed(Server) end, %% Delete any leftover compaction files. If we don't do this a @@ -490,10 +478,7 @@ handle_call({db_updated, #db{}=Db}, _From, Server0) -> Server = try ets:lookup_element(couch_dbs, DbName, #db.instance_start_time) of StartTime -> true = ets:insert(couch_dbs, Db), - Lru = case couch_db:is_system_db(Db) of - false -> couch_lru:update(DbName, Server0#server.lru); - true -> Server0#server.lru - end, + Lru = couch_lru:update(DbName, Server0#server.lru), Server0#server{lru = Lru}; _ -> Server0 @@ -532,7 +517,7 @@ handle_info({'EXIT', Pid, Reason}, Server) -> end, true = ets:delete(couch_dbs, DbName), true = ets:delete(couch_dbs_pid_to_name, Pid), - {noreply, db_closed(Server, Db#db.options)}; + {noreply, db_closed(Server)}; [] -> {noreply, Server} end; @@ -542,17 +527,11 @@ handle_info(restart_config_listener, State) -> handle_info(Info, Server) -> {stop, {unknown_message, Info}, Server}. -db_opened(Server, Options) -> - case lists:member(sys_db, Options) of - false -> Server#server{dbs_open=Server#server.dbs_open + 1}; - true -> Server - end. +db_opened(Server) -> + Server#server{dbs_open=Server#server.dbs_open + 1}. -db_closed(Server, Options) -> - case lists:member(sys_db, Options) of - false -> Server#server{dbs_open=Server#server.dbs_open - 1}; - true -> Server - end. +db_closed(Server) -> + Server#server{dbs_open=Server#server.dbs_open - 1}. -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/92c25a98/test/couchdb_views_tests.erl ---------------------------------------------------------------------- diff --git a/test/couchdb_views_tests.erl b/test/couchdb_views_tests.erl index 7b04e85..f1fddfc 100644 --- a/test/couchdb_views_tests.erl +++ b/test/couchdb_views_tests.erl @@ -372,7 +372,6 @@ couchdb_1283() -> MonRef = erlang:monitor(process, CPid), Writer3 = spawn_writer(Db3#db.name), ?assert(is_process_alive(Writer3)), - ?assertEqual({error, all_dbs_active}, get_writer_status(Writer3)), ?assert(is_process_alive(Writer1)), ?assert(is_process_alive(Writer2)), @@ -391,7 +390,6 @@ couchdb_1283() -> {reason, "Failure compacting view group"}]}) end, - ?assertEqual(ok, writer_try_again(Writer3)), ?assertEqual(ok, get_writer_status(Writer3)), ?assert(is_process_alive(Writer1)),