Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 71DD5200C4C for ; Tue, 4 Apr 2017 23:33:00 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 70A1E160BA2; Tue, 4 Apr 2017 21:33:00 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 133D4160BAE for ; Tue, 4 Apr 2017 23:32:58 +0200 (CEST) Received: (qmail 34416 invoked by uid 500); 4 Apr 2017 21:32:57 -0000 Mailing-List: contact commits-help@couchdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@couchdb.apache.org Delivered-To: mailing list commits@couchdb.apache.org Received: (qmail 33863 invoked by uid 99); 4 Apr 2017 21:32:57 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 04 Apr 2017 21:32:57 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 4B9F1DFDAC; Tue, 4 Apr 2017 21:32:57 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: davisp@apache.org To: commits@couchdb.apache.org Date: Tue, 04 Apr 2017 21:33:19 -0000 Message-Id: <171c105b3f8246efbdd5706cdfb26f32@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [24/26] couch commit: updated refs/heads/COUCHDB-3288-remove-public-db-record to c515bca archived-at: Tue, 04 Apr 2017 21:33:00 -0000 Update couch_server to not use the db record This removes introspection of the #db record by couch_server. While its required for the pluggable storage engine upgrade, its also nice to remove the hacky overloading of #db record fields for couch_server logic. COUCHDB-3288 Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch/commit/73c273ff Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch/tree/73c273ff Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch/diff/73c273ff Branch: refs/heads/COUCHDB-3288-remove-public-db-record Commit: 73c273ff93807cb845ccacd769d0ab7e1030b69d Parents: e1491f1 Author: Paul J. Davis Authored: Fri Feb 3 10:20:30 2017 -0600 Committer: Paul J. Davis Committed: Tue Apr 4 16:32:27 2017 -0500 ---------------------------------------------------------------------- src/couch_db.erl | 23 +++++++++ src/couch_lru.erl | 14 +++--- src/couch_server.erl | 109 ++++++++++++++++++++++-------------------- src/couch_server_int.hrl | 23 +++++++++ 4 files changed, 110 insertions(+), 59 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/73c273ff/src/couch_db.erl ---------------------------------------------------------------------- diff --git a/src/couch_db.erl b/src/couch_db.erl index 1f68200..3a29a3d 100644 --- a/src/couch_db.erl +++ b/src/couch_db.erl @@ -19,6 +19,9 @@ reopen/1, close/1, + incref/1, + decref/1, + monitor/1, monitored_by/1, is_idle/1, @@ -34,7 +37,9 @@ get_db_info/1, get_doc_count/1, get_epochs/1, + get_instance_start_time/1, get_last_purged/1, + get_pid/1, get_revs_limit/1, get_security/1, get_update_seq/1, @@ -46,6 +51,7 @@ increment_update_seq/1, set_revs_limit/2, set_security/2, + set_user_ctx/2, ensure_full_commit/1, ensure_full_commit/2, @@ -181,6 +187,14 @@ reopen(#db{main_pid = Pid, fd = Fd, fd_monitor = OldRef, user_ctx = UserCtx}) -> {ok, NewDb#db{user_ctx = UserCtx, fd_monitor = NewRef}} end. +incref(#db{fd = Fd} = Db) -> + Ref = erlang:monitor(process, Fd), + {ok, Db#db{fd_monitor = Ref}}. + +decref(#db{fd_monitor = Monitor}) -> + erlang:demonitor(Monitor, [flush]), + ok. + is_system_db(#db{options = Options}) -> lists:member(sys_db, Options). @@ -381,6 +395,9 @@ get_last_purged(#db{}=Db) -> couch_file:pread_term(Db#db.fd, Pointer) end. +get_pid(#db{main_pid = Pid}) -> + Pid. + get_doc_count(Db) -> {ok, {Count, _, _}} = couch_btree:full_reduce(Db#db.id_tree), {ok, Count}. @@ -393,6 +410,9 @@ get_epochs(#db{}=Db) -> validate_epochs(Epochs), Epochs. +get_instance_start_time(#db{instance_start_time = IST}) -> + IST. + get_compacted_seq(#db{}=Db) -> couch_db_header:compacted_seq(Db#db.header). @@ -585,6 +605,9 @@ set_security(#db{main_pid=Pid}=Db, {NewSecProps}) when is_list(NewSecProps) -> set_security(_, _) -> throw(bad_request). +set_user_ctx(#db{} = Db, UserCtx) -> + {ok, Db#db{user_ctx = UserCtx}}. + validate_security_object(SecProps) -> Admins = couch_util:get_value(<<"admins">>, SecProps, {[]}), % we fallback to readers here for backwards compatibility http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/73c273ff/src/couch_lru.erl ---------------------------------------------------------------------- diff --git a/src/couch_lru.erl b/src/couch_lru.erl index b79286e..4bf0409 100644 --- a/src/couch_lru.erl +++ b/src/couch_lru.erl @@ -13,7 +13,7 @@ -module(couch_lru). -export([new/0, insert/2, update/2, close/1]). --include_lib("couch/include/couch_db.hrl"). +-include("couch_server_int.hrl"). new() -> Updates = ets:new(couch_lru_updates, [ordered_set]), @@ -49,18 +49,20 @@ close({Count, Updates, Dbs}) -> close_int('$end_of_table', _Updates, _Dbs) -> false; close_int({_Count, DbName} = Key, Updates, Dbs) -> - case ets:update_element(couch_dbs, DbName, {#db.fd_monitor, locked}) of + case ets:update_element(couch_dbs, DbName, {#entry.lock, locked}) of true -> - [#db{main_pid = Pid} = Db] = ets:lookup(couch_dbs, DbName), + [#entry{db = Db}] = ets:lookup(couch_dbs, DbName), case couch_db:is_idle(Db) of true -> + DbPid = couch_db:get_pid(Db), true = ets:delete(couch_dbs, DbName), - true = ets:delete(couch_dbs_pid_to_name, Pid), - exit(Pid, kill), + true = ets:delete(couch_dbs_pid_to_name, DbPid), + exit(DbPid, kill), true = ets:delete(Updates, Key), true = ets:delete(Dbs, DbName), true; false -> - true = ets:update_element(couch_dbs, DbName, {#db.fd_monitor, nil}), + ElemSpec = {#entry.lock, unlocked}, + true = ets:update_element(couch_dbs, DbName, ElemSpec), couch_stats:increment_counter([couchdb, couch_server, lru_skip]), close_int(ets:next(Updates, Key), Updates, Dbs) end; http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/73c273ff/src/couch_server.erl ---------------------------------------------------------------------- diff --git a/src/couch_server.erl b/src/couch_server.erl index 1152300..0513f63 100644 --- a/src/couch_server.erl +++ b/src/couch_server.erl @@ -26,6 +26,7 @@ -export([handle_config_change/5, handle_config_terminate/3]). -include_lib("couch/include/couch_db.hrl"). +-include("couch_server_int.hrl"). -define(MAX_DBS_OPEN, 100). -define(RELISTEN_DELAY, 5000). @@ -73,16 +74,18 @@ sup_start_link() -> open(DbName, Options0) -> Ctx = couch_util:get_value(user_ctx, Options0, #user_ctx{}), case ets:lookup(couch_dbs, DbName) of - [#db{fd=Fd, fd_monitor=Lock} = Db] when Lock =/= locked -> + [#entry{db = Db0, lock = Lock}] when Lock =/= locked -> update_lru(DbName), - {ok, Db#db{user_ctx=Ctx, fd_monitor=erlang:monitor(process,Fd)}}; + {ok, Db1} = couch_db:incref(Db0), + couch_db:set_user_ctx(Db1, Ctx); _ -> Options = maybe_add_sys_db_callbacks(DbName, Options0), Timeout = couch_util:get_value(timeout, Options, infinity), Create = couch_util:get_value(create_if_missing, Options, false), case gen_server:call(couch_server, {open, DbName, Options}, Timeout) of - {ok, #db{fd=Fd} = Db} -> - {ok, Db#db{user_ctx=Ctx, fd_monitor=erlang:monitor(process,Fd)}}; + {ok, Db0} -> + {ok, Db1} = couch_db:incref(Db0), + couch_db:set_user_ctx(Db1, Ctx); {not_found, no_db_file} when Create -> couch_log:warning("creating missing database: ~s", [DbName]), couch_server:create(DbName, Options); @@ -100,9 +103,10 @@ close_lru() -> create(DbName, Options0) -> Options = maybe_add_sys_db_callbacks(DbName, Options0), case gen_server:call(couch_server, {create, DbName, Options}, infinity) of - {ok, #db{fd=Fd} = Db} -> + {ok, Db0} -> Ctx = couch_util:get_value(user_ctx, Options, #user_ctx{}), - {ok, Db#db{user_ctx=Ctx, fd_monitor=erlang:monitor(process,Fd)}}; + {ok, Db1} = couch_db:incref(Db0), + couch_db:set_user_ctx(Db1, Ctx); Error -> Error end. @@ -184,7 +188,7 @@ init([]) -> ok = config:listen_for_changes(?MODULE, nil), ok = couch_file:init_delete_dir(RootDir), hash_admin_passwords(), - ets:new(couch_dbs, [set, protected, named_table, {keypos, #db.name}]), + ets:new(couch_dbs, [set, protected, named_table, {keypos, #entry.name}]), ets:new(couch_dbs_pid_to_name, [set, protected, named_table]), process_flag(trap_exit, true), {ok, #server{root_dir=RootDir, @@ -196,8 +200,9 @@ terminate(Reason, Srv) -> couch_log:error("couch_server terminating with ~p, state ~2048p", [Reason, Srv#server{lru = redacted}]), - ets:foldl(fun(#db{main_pid=Pid}, _) -> couch_util:shutdown_sync(Pid) end, - nil, couch_dbs), + ets:foldl(fun(Db, _) -> + couch_util:shutdown_sync(couch_db:get_pid(Db)) + end, nil, couch_dbs), ok. handle_config_change("couchdb", "database_dir", _, _, _) -> @@ -297,15 +302,13 @@ open_async(Server, From, DbName, Filepath, Options) -> true -> create; false -> open end, - % icky hack of field values - compactor_pid used to store clients - % and fd used for opening request info - true = ets:insert(couch_dbs, #db{ + true = ets:insert(couch_dbs, #entry{ name = DbName, - fd = ReqType, - main_pid = Opener, - compactor_pid = [From], - fd_monitor = locked, - options = Options + pid = Opener, + lock = locked, + waiters = [From], + req_type = ReqType, + db_options = Options }), true = ets:insert(couch_dbs_pid_to_name, {Opener, DbName}), db_opened(Server). @@ -329,16 +332,15 @@ handle_call({open_result, T0, DbName, {ok, Db}}, {FromPid, _Tag}, Server) -> true = ets:delete(couch_dbs_pid_to_name, FromPid), OpenTime = timer:now_diff(os:timestamp(), T0) / 1000, couch_stats:update_histogram([couchdb, db_open_time], OpenTime), - % icky hack of field values - compactor_pid used to store clients - % and fd used to possibly store a creation request + DbPid = couch_db:get_pid(Db), case ets:lookup(couch_dbs, DbName) of [] -> % db was deleted during async open - exit(Db#db.main_pid, kill), + exit(DbPid, kill), {reply, ok, Server}; - [#db{fd=ReqType, compactor_pid=Froms}] -> - link(Db#db.main_pid), - [gen_server:reply(From, {ok, Db}) || From <- Froms], + [#entry{req_type = ReqType, waiters = Waiters} = Entry] -> + link(DbPid), + [gen_server:reply(Waiter, {ok, Db}) || Waiter <- Waiters], % Cancel the creation request if it exists. case ReqType of {create, DbName, _Filepath, _Options, CrFrom} -> @@ -346,21 +348,27 @@ handle_call({open_result, T0, DbName, {ok, Db}}, {FromPid, _Tag}, Server) -> _ -> ok end, - true = ets:insert(couch_dbs, Db), - true = ets:insert(couch_dbs_pid_to_name, {Db#db.main_pid, DbName}), + true = ets:insert(couch_dbs, #entry{ + name = DbName, + db = Db, + pid = DbPid, + lock = unlocked, + db_options = Entry#entry.db_options, + start_time = couch_db:get_instance_start_time(Db) + }), + true = ets:insert(couch_dbs_pid_to_name, {DbPid, DbName}), Lru = couch_lru:insert(DbName, Server#server.lru), {reply, ok, Server#server{lru = Lru}} end; handle_call({open_result, T0, DbName, {error, eexist}}, From, Server) -> handle_call({open_result, T0, DbName, file_exists}, From, Server); handle_call({open_result, _T0, DbName, Error}, {FromPid, _Tag}, Server) -> - % icky hack of field values - compactor_pid used to store clients case ets:lookup(couch_dbs, DbName) of [] -> % db was deleted during async open {reply, ok, Server}; - [#db{fd=ReqType, compactor_pid=Froms}] -> - [gen_server:reply(From, Error) || From <- Froms], + [#entry{req_type = ReqType, waiters = Waiters} = Entry] -> + [gen_server:reply(Waiter, Error) || Waiter <- Waiters], couch_log:info("open_result error ~p for ~s", [Error, DbName]), true = ets:delete(couch_dbs, DbName), true = ets:delete(couch_dbs_pid_to_name, FromPid), @@ -384,15 +392,14 @@ handle_call({open, DbName, Options}, From, Server) -> Error -> {reply, Error, Server} end; - [#db{compactor_pid = Froms} = Db] when is_list(Froms) -> - % icky hack of field values - compactor_pid used to store clients - true = ets:insert(couch_dbs, Db#db{compactor_pid = [From|Froms]}), - if length(Froms) =< 10 -> ok; true -> + [#entry{waiters = Waiters} = Entry] when is_list(Waiters) -> + true = ets:insert(couch_dbs, Entry#entry{waiters = [From | Waiters]}), + if length(Waiters) =< 10 -> ok; true -> Fmt = "~b clients waiting to open db ~s", - couch_log:info(Fmt, [length(Froms), DbName]) + couch_log:info(Fmt, [length(Waiters), DbName]) end, {noreply, Server}; - [#db{} = Db] -> + [#entry{db = Db}] -> {reply, {ok, Db}, Server} end; handle_call({create, DbName, Options}, From, Server) -> @@ -405,14 +412,13 @@ handle_call({create, DbName, Options}, From, Server) -> {ok, Server2} = maybe_close_lru_db(Server), {noreply, open_async(Server2, From, DbName, Filepath, [create | Options])}; - [#db{fd=open}=Db] -> + [#entry{req_type = open} = Entry] -> % We're trying to create a database while someone is in % the middle of trying to open it. We allow one creator % to wait while we figure out if it'll succeed. - % icky hack of field values - fd used to store create request CrOptions = [create | Options], - NewDb = Db#db{fd={create, DbName, Filepath, CrOptions, From}}, - true = ets:insert(couch_dbs, NewDb), + Req = {create, DbName, Filepath, CrOptions, From}, + true = ets:insert(couch_dbs, Entry#entry{req_type = Req}), {noreply, Server}; [_AlreadyRunningDb] -> {reply, file_exists, Server} @@ -428,18 +434,17 @@ handle_call({delete, DbName, Options}, _From, Server) -> Server2 = case ets:lookup(couch_dbs, DbName) of [] -> Server; - [#db{main_pid=Pid, compactor_pid=Froms}] when is_list(Froms) -> - % icky hack of field values - compactor_pid used to store clients + [#entry{pid = Pid, waiters = Waiters} = Entry] when is_list(Waiters) -> true = ets:delete(couch_dbs, DbName), true = ets:delete(couch_dbs_pid_to_name, Pid), exit(Pid, kill), - [gen_server:reply(F, not_found) || F <- Froms], + [gen_server:reply(Waiter, not_found) || Waiter <- Waiters], db_closed(Server); - [#db{main_pid=Pid}] -> + [#entry{pid = Pid} = Entry] -> true = ets:delete(couch_dbs, DbName), true = ets:delete(couch_dbs_pid_to_name, Pid), exit(Pid, kill), - db_closed(Server) + db_closed(Server); end, %% Delete any leftover compaction files. If we don't do this a @@ -465,11 +470,12 @@ handle_call({delete, DbName, Options}, _From, Server) -> Error -> {reply, Error, Server} end; -handle_call({db_updated, #db{}=Db}, _From, Server0) -> - #db{name = DbName, instance_start_time = StartTime} = Db, - Server = try ets:lookup_element(couch_dbs, DbName, #db.instance_start_time) of +handle_call({db_updated, Db}, _From, Server0) -> + DbName = couch_db:name(Db), + StartTime = couch_db:get_instance_start_time(Db), + Server = try ets:lookup_element(couch_dbs, DbName, #entry.start_time) of StartTime -> - true = ets:insert(couch_dbs, Db), + true = ets:update_element(couch_dbs, DbName, {#entry.db, Db}), Lru = couch_lru:update(DbName, Server0#server.lru), Server0#server{lru = Lru}; _ -> @@ -494,18 +500,15 @@ handle_info({'EXIT', _Pid, config_change}, Server) -> handle_info({'EXIT', Pid, Reason}, Server) -> case ets:lookup(couch_dbs_pid_to_name, Pid) of [{Pid, DbName}] -> - [#db{compactor_pid=Froms}] = ets:lookup(couch_dbs, DbName), + [#entry{waiters = Waiters} = Entry] = ets:lookup(couch_dbs, DbName), if Reason /= snappy_nif_not_loaded -> ok; true -> Msg = io_lib:format("To open the database `~s`, Apache CouchDB " "must be built with Erlang OTP R13B04 or higher.", [DbName]), couch_log:error(Msg, []) end, couch_log:info("db ~s died with reason ~p", [DbName, Reason]), - % icky hack of field values - compactor_pid used to store clients - if is_list(Froms) -> - [gen_server:reply(From, Reason) || From <- Froms]; - true -> - ok + if not is_list(Waiters) -> ok; true -> + [gen_server:reply(Waiter, Reason) || Waiter <- Waiters] end, true = ets:delete(couch_dbs, DbName), true = ets:delete(couch_dbs_pid_to_name, Pid), http://git-wip-us.apache.org/repos/asf/couchdb-couch/blob/73c273ff/src/couch_server_int.hrl ---------------------------------------------------------------------- diff --git a/src/couch_server_int.hrl b/src/couch_server_int.hrl new file mode 100644 index 0000000..537a6ab --- /dev/null +++ b/src/couch_server_int.hrl @@ -0,0 +1,23 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + + +-record(entry, { + name, + db, + pid, + lock, + waiters, + req_type, + db_options, + start_time +}).