Return-Path: Delivered-To: apmail-couchdb-commits-archive@www.apache.org Received: (qmail 26433 invoked from network); 4 Jul 2009 15:48:04 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 4 Jul 2009 15:48:04 -0000 Received: (qmail 24983 invoked by uid 500); 4 Jul 2009 15:48:15 -0000 Delivered-To: apmail-couchdb-commits-archive@couchdb.apache.org Received: (qmail 24911 invoked by uid 500); 4 Jul 2009 15:48:15 -0000 Mailing-List: contact commits-help@couchdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@couchdb.apache.org Delivered-To: mailing list commits@couchdb.apache.org Received: (qmail 24902 invoked by uid 99); 4 Jul 2009 15:48:15 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 04 Jul 2009 15:48:15 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 04 Jul 2009 15:48:08 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 6AB0723888D7; Sat, 4 Jul 2009 15:47:48 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r791140 - in /couchdb/trunk: etc/couchdb/ share/www/script/ share/www/script/test/ src/couchdb/ Date: Sat, 04 Jul 2009 15:47:47 -0000 To: commits@couchdb.apache.org From: jchris@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20090704154748.6AB0723888D7@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: jchris Date: Sat Jul 4 15:47:47 2009 New Revision: 791140 URL: http://svn.apache.org/viewvc?rev=791140&view=rev Log: Name view index files by their function hashes for no downtime deploys. Closes COUCHDB-218 Adds ability to switch view indexes on the fly by building the index from a "staging" design doc, and then COPYing the staging doc to the production doc's id. Since indexes are referenced by view definition, the new version of the production design doc will point immediately to the index files already built in staging. Please use and give feedback. Modified: couchdb/trunk/etc/couchdb/default.ini.tpl.in couchdb/trunk/share/www/script/couch.js couchdb/trunk/share/www/script/test/design_docs.js couchdb/trunk/src/couchdb/couch_db.erl couchdb/trunk/src/couchdb/couch_db.hrl couchdb/trunk/src/couchdb/couch_db_updater.erl couchdb/trunk/src/couchdb/couch_httpd_db.erl couchdb/trunk/src/couchdb/couch_task_status.erl couchdb/trunk/src/couchdb/couch_view.erl couchdb/trunk/src/couchdb/couch_view_group.erl couchdb/trunk/src/couchdb/couch_view_updater.erl Modified: couchdb/trunk/etc/couchdb/default.ini.tpl.in URL: http://svn.apache.org/viewvc/couchdb/trunk/etc/couchdb/default.ini.tpl.in?rev=791140&r1=791139&r2=791140&view=diff ============================================================================== --- couchdb/trunk/etc/couchdb/default.ini.tpl.in (original) +++ couchdb/trunk/etc/couchdb/default.ini.tpl.in Sat Jul 4 15:47:47 2009 @@ -65,6 +65,7 @@ _whoami = {couch_httpd_misc_handlers, handle_whoami_req} [httpd_db_handlers] +_view_cleanup = {couch_httpd_db, handle_view_cleanup_req} _compact = {couch_httpd_db, handle_compact_req} _design = {couch_httpd_db, handle_design_req} _temp_view = {couch_httpd_view, handle_temp_view_req} @@ -80,3 +81,4 @@ _view = {couch_httpd_view, handle_view_req} _show = {couch_httpd_show, handle_doc_show_req} _list = {couch_httpd_show, handle_view_list_req} +_info = {couch_httpd_db, handle_design_info_req} Modified: couchdb/trunk/share/www/script/couch.js URL: http://svn.apache.org/viewvc/couchdb/trunk/share/www/script/couch.js?rev=791140&r1=791139&r2=791140&view=diff ============================================================================== --- couchdb/trunk/share/www/script/couch.js [utf-8] (original) +++ couchdb/trunk/share/www/script/couch.js [utf-8] Sat Jul 4 15:47:47 2009 @@ -177,6 +177,19 @@ return JSON.parse(this.last_req.responseText); } + // gets information about a design doc + this.designInfo = function(docid) { + this.last_req = this.request("GET", this.uri + docid + "/_info"); + CouchDB.maybeThrowError(this.last_req); + return JSON.parse(this.last_req.responseText); + } + + this.viewCleanup = function() { + this.last_req = this.request("POST", this.uri + "_view_cleanup"); + CouchDB.maybeThrowError(this.last_req); + return JSON.parse(this.last_req.responseText); + } + this.allDocs = function(options,keys) { if(!keys) { this.last_req = this.request("GET", this.uri + "_all_docs" + encodeOptions(options)); Modified: couchdb/trunk/share/www/script/test/design_docs.js URL: http://svn.apache.org/viewvc/couchdb/trunk/share/www/script/test/design_docs.js?rev=791140&r1=791139&r2=791140&view=diff ============================================================================== --- couchdb/trunk/share/www/script/test/design_docs.js (original) +++ couchdb/trunk/share/www/script/test/design_docs.js Sat Jul 4 15:47:47 2009 @@ -49,6 +49,14 @@ } T(db.save(designDoc).ok); + // test that we get design doc info back + var dinfo = db.designInfo("_design/test"); + TEquals("test", dinfo.name); + var vinfo = dinfo.view_index; + TEquals(51, vinfo.disk_size); + TEquals(false, vinfo.compact_running); + TEquals("64625dce94960fd5ca116e42aa9d011a", vinfo.signature); + db.bulkSave(makeDocs(1, numDocs + 1)); // test that the _all_docs view returns correctly with keys @@ -66,7 +74,7 @@ T(db.ensureFullCommit().ok); restartServer(); }; - + // test when language not specified, Javascript is implied var designDoc2 = { _id:"_design/test2", @@ -110,5 +118,9 @@ restartServer(); T(db.open(designDoc._id) == null); T(db.view("test/no_docs") == null); + + // trigger ddoc cleanup + T(db.viewCleanup().ok); + }); }; Modified: couchdb/trunk/src/couchdb/couch_db.erl URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_db.erl?rev=791140&r1=791139&r2=791140&view=diff ============================================================================== --- couchdb/trunk/src/couchdb/couch_db.erl (original) +++ couchdb/trunk/src/couchdb/couch_db.erl Sat Jul 4 15:47:47 2009 @@ -13,7 +13,7 @@ -module(couch_db). -behaviour(gen_server). --export([open/2,close/1,create/2,start_compact/1,get_db_info/1]). +-export([open/2,close/1,create/2,start_compact/1,get_db_info/1,get_design_docs/1]). -export([open_ref_counted/2,is_idle/1,monitor/1,count_changes_since/2]). -export([update_doc/3,update_docs/4,update_docs/2,update_docs/3,delete_doc/3]). -export([get_doc_info/2,open_doc/2,open_doc/3,open_doc_revs/4]). @@ -192,6 +192,16 @@ ], {ok, InfoList}. +get_design_docs(#db{fulldocinfo_by_id_btree=Btree}=Db) -> + couch_btree:foldl(Btree, <<"_design/">>, + fun(#full_doc_info{id= <<"_design/",_/binary>>}=FullDocInfo, _Reds, AccDocs) -> + {ok, Doc} = couch_db:open_doc_int(Db, FullDocInfo, []), + {ok, [Doc | AccDocs]}; + (_, _Reds, AccDocs) -> + {stop, AccDocs} + end, + []). + check_is_admin(#db{admins=Admins, user_ctx=#user_ctx{name=Name,roles=Roles}}) -> DbAdmins = [<<"_admin">> | Admins], case DbAdmins -- [Name | Roles] of Modified: couchdb/trunk/src/couchdb/couch_db.hrl URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_db.hrl?rev=791140&r1=791139&r2=791140&view=diff ============================================================================== --- couchdb/trunk/src/couchdb/couch_db.hrl (original) +++ couchdb/trunk/src/couchdb/couch_db.hrl Sat Jul 4 15:47:47 2009 @@ -190,8 +190,7 @@ headers = [] }). --record(group, - {type=view, % can also be temp_view +-record(group, { sig=nil, db=nil, fd=nil, Modified: couchdb/trunk/src/couchdb/couch_db_updater.erl URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_db_updater.erl?rev=791140&r1=791139&r2=791140&view=diff ============================================================================== --- couchdb/trunk/src/couchdb/couch_db_updater.erl (original) +++ couchdb/trunk/src/couchdb/couch_db_updater.erl Sat Jul 4 15:47:47 2009 @@ -350,7 +350,7 @@ refresh_validate_doc_funs(Db) -> - {ok, DesignDocs} = get_design_docs(Db), + {ok, DesignDocs} = couch_db:get_design_docs(Db), ProcessDocFuns = lists:flatmap( fun(DesignDoc) -> case couch_doc:get_validate_doc_fun(DesignDoc) of @@ -360,16 +360,6 @@ end, DesignDocs), Db#db{validate_doc_funs=ProcessDocFuns}. -get_design_docs(#db{fulldocinfo_by_id_btree=Btree}=Db) -> - couch_btree:foldl(Btree, <<"_design/">>, - fun(#full_doc_info{id= <<"_design/",_/binary>>}=FullDocInfo, _Reds, AccDocs) -> - {ok, Doc} = couch_db:open_doc_int(Db, FullDocInfo, []), - {ok, [Doc | AccDocs]}; - (_, _Reds, AccDocs) -> - {stop, AccDocs} - end, - []). - % rev tree functions flush_trees(_Db, [], AccFlushedTrees) -> Modified: couchdb/trunk/src/couchdb/couch_httpd_db.erl URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_httpd_db.erl?rev=791140&r1=791139&r2=791140&view=diff ============================================================================== --- couchdb/trunk/src/couchdb/couch_httpd_db.erl (original) +++ couchdb/trunk/src/couchdb/couch_httpd_db.erl Sat Jul 4 15:47:47 2009 @@ -15,7 +15,8 @@ -export([handle_request/1, handle_compact_req/2, handle_design_req/2, db_req/2, couch_doc_open/4,handle_changes_req/2, - update_doc_result_to_json/1, update_doc_result_to_json/2]). + update_doc_result_to_json/1, update_doc_result_to_json/2, + handle_design_info_req/2, handle_view_cleanup_req/2]). -import(couch_httpd, [send_json/2,send_json/3,send_json/4,send_method_not_allowed/2, @@ -156,6 +157,15 @@ handle_compact_req(Req, _Db) -> send_method_not_allowed(Req, "POST"). +handle_view_cleanup_req(#httpd{method='POST'}=Req, Db) -> + % delete unreferenced index files + ok = couch_view:cleanup_index_files(Db), + send_json(Req, 202, {[{ok, true}]}); + +handle_view_cleanup_req(Req, _Db) -> + send_method_not_allowed(Req, "POST"). + + handle_design_req(#httpd{ path_parts=[_DbName,_Design,_DesName, <<"_",_/binary>> = Action | _Rest], design_url_handlers = DesignUrlHandlers @@ -166,6 +176,23 @@ handle_design_req(Req, Db) -> db_req(Req, Db). +handle_design_info_req(#httpd{ + method='GET', + path_parts=[_DbName, _Design, DesignName, _] + }=Req, Db) -> + DesignId = <<"_design/", DesignName/binary>>, + ?LOG_ERROR("DesignId ~p",[DesignId]), + {ok, GroupInfoList} = couch_view:get_group_info(Db, DesignId), + ?LOG_ERROR("GroupInfoList ~p",[GroupInfoList]), + send_json(Req, 200, {[ + {name, DesignName}, + {view_index, {GroupInfoList}} + ]}); + +handle_design_info_req(Req, _Db) -> + send_method_not_allowed(Req, "GET"). + + create_db_req(#httpd{user_ctx=UserCtx}=Req, DbName) -> ok = couch_httpd:verify_is_server_admin(Req), case couch_server:create(DbName, [{user_ctx, UserCtx}]) of Modified: couchdb/trunk/src/couchdb/couch_task_status.erl URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_task_status.erl?rev=791140&r1=791139&r2=791140&view=diff ============================================================================== --- couchdb/trunk/src/couchdb/couch_task_status.erl (original) +++ couchdb/trunk/src/couchdb/couch_task_status.erl Sat Jul 4 15:47:47 2009 @@ -107,12 +107,12 @@ handle_cast({update_status, Pid, StatusText}, Server) -> [{Pid, {Type, TaskName, _StatusText}}] = ets:lookup(?MODULE, Pid), + ?LOG_DEBUG("New task status for ~s: ~s",[TaskName, StatusText]), true = ets:insert(?MODULE, {Pid, {Type, TaskName, StatusText}}), {noreply, Server}; handle_cast(stop, State) -> {stop, normal, State}. - handle_info({'DOWN', _MonitorRef, _Type, Pid, _Info}, Server) -> %% should we also erlang:demonitor(_MonitorRef), ? ets:delete(?MODULE, Pid), Modified: couchdb/trunk/src/couchdb/couch_view.erl URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_view.erl?rev=791140&r1=791139&r2=791140&view=diff ============================================================================== --- couchdb/trunk/src/couchdb/couch_view.erl (original) +++ couchdb/trunk/src/couchdb/couch_view.erl Sat Jul 4 15:47:47 2009 @@ -17,7 +17,7 @@ detuple_kvs/2,init/1,terminate/2,handle_call/3,handle_cast/2,handle_info/2, code_change/3,get_reduce_view/4,get_temp_reduce_view/5,get_temp_map_view/4, get_map_view/4,get_row_count/1,reduce_to_count/1,fold_reduce/7, - extract_map_view/1,get_group_server/2]). + extract_map_view/1,get_group_server/2,get_group_info/2,cleanup_index_files/1]). -include("couch_db.hrl"). @@ -28,18 +28,32 @@ start_link() -> gen_server:start_link({local, couch_view}, couch_view, [], []). -get_temp_updater(DbName, Type, DesignOptions, MapSrc, RedSrc) -> - {ok, Pid} = gen_server:call(couch_view, - {start_temp_updater, DbName, Type, DesignOptions, MapSrc, RedSrc}), - Pid. - -get_group_server(DbName, GroupId) -> - case gen_server:call(couch_view, {start_group_server, DbName, GroupId}) of +get_temp_updater(DbName, Language, DesignOptions, MapSrc, RedSrc) -> + % make temp group + % do we need to close this db? + {ok, _Db, Group} = + couch_view_group:open_temp_group(DbName, Language, DesignOptions, MapSrc, RedSrc), + case gen_server:call(couch_view, {get_group_server, DbName, Group}) of {ok, Pid} -> Pid; Error -> throw(Error) end. + +get_group_server(DbName, GroupId) -> + % get signature for group + case couch_view_group:open_db_group(DbName, GroupId) of + % do we need to close this db? + {ok, _Db, Group} -> + case gen_server:call(couch_view, {get_group_server, DbName, Group}) of + {ok, Pid} -> + Pid; + Error -> + throw(Error) + end; + Error -> + throw(Error) + end. get_group(Db, GroupId, Stale) -> MinUpdateSeq = case Stale of @@ -50,18 +64,52 @@ get_group_server(couch_db:name(Db), GroupId), MinUpdateSeq). - -get_temp_group(Db, Type, DesignOptions, MapSrc, RedSrc) -> +get_temp_group(Db, Language, DesignOptions, MapSrc, RedSrc) -> couch_view_group:request_group( - get_temp_updater(couch_db:name(Db), Type, DesignOptions, MapSrc, RedSrc), - couch_db:get_update_seq(Db)). + get_temp_updater(couch_db:name(Db), Language, DesignOptions, MapSrc, RedSrc), + couch_db:get_update_seq(Db)). + +get_group_info(Db, GroupId) -> + couch_view_group:request_group_info( + get_group_server(couch_db:name(Db), GroupId)). + +cleanup_index_files(Db) -> + % load all ddocs + {ok, DesignDocs} = couch_db:get_design_docs(Db), + + % make unique list of group sigs + Sigs = lists:map(fun(#doc{id = GroupId} = DDoc) -> + {ok, Info} = get_group_info(Db, GroupId), + ?b2l(proplists:get_value(signature, Info)) + end, [DD||DD <- DesignDocs, DD#doc.deleted == false]), + + FileList = list_index_files(Db), + + % regex that matches all ddocs + RegExp = "("++ string:join(Sigs, "|") ++")", + + % filter out the ones in use + DeleteFiles = lists:filter(fun(FilePath) -> + regexp:first_match(FilePath, RegExp)==nomatch + end, FileList), + % delete unused files + ?LOG_DEBUG("deleting unused view index files: ~p",[DeleteFiles]), + [file:delete(File)||File <- DeleteFiles], + ok. + +list_index_files(Db) -> + % call server to fetch the index files + RootDir = couch_config:get("couchdb", "view_index_dir"), + Files = filelib:wildcard(RootDir ++ "/." ++ ?b2l(couch_db:name(Db)) ++ "_design"++"/*"). + get_row_count(#view{btree=Bt}) -> {ok, {Count, _Reds}} = couch_btree:full_reduce(Bt), {ok, Count}. -get_temp_reduce_view(Db, Type, DesignOptions, MapSrc, RedSrc) -> - {ok, #group{views=[View]}=Group} = get_temp_group(Db, Type, DesignOptions, MapSrc, RedSrc), +get_temp_reduce_view(Db, Language, DesignOptions, MapSrc, RedSrc) -> + {ok, #group{views=[View]}=Group} = + get_temp_group(Db, Language, DesignOptions, MapSrc, RedSrc), {ok, {temp_reduce, View}, Group}. @@ -141,8 +189,8 @@ get_key_pos(Key, Rest, N+1). -get_temp_map_view(Db, Type, DesignOptions, Src) -> - {ok, #group{views=[View]}=Group} = get_temp_group(Db, Type, DesignOptions, Src, []), +get_temp_map_view(Db, Language, DesignOptions, Src) -> + {ok, #group{views=[View]}=Group} = get_temp_group(Db, Language, DesignOptions, Src, []), {ok, View, Group}. get_map_view(Db, GroupId, Name, Stale) -> @@ -220,9 +268,8 @@ ok end), ets:new(couch_groups_by_db, [bag, private, named_table]), - ets:new(group_servers_by_name, [set, protected, named_table]), + ets:new(group_servers_by_sig, [set, protected, named_table]), ets:new(couch_groups_by_updater, [set, private, named_table]), - ets:new(couch_temp_group_fd_by_db, [set, protected, named_table]), process_flag(trap_exit, true), {ok, #server{root_dir=RootDir}}. @@ -232,37 +279,15 @@ ok. -handle_call({start_temp_updater, DbName, Lang, DesignOptions, MapSrc, RedSrc}, - _From, #server{root_dir=Root}=Server) -> - <> = erlang:md5(term_to_binary({Lang, DesignOptions, MapSrc, RedSrc})), - Name = lists:flatten(io_lib:format("_temp_~.36B",[SigInt])), - Pid = - case ets:lookup(group_servers_by_name, {DbName, Name}) of +handle_call({get_group_server, DbName, + #group{name=GroupId,sig=Sig}=Group}, _From, #server{root_dir=Root}=Server) -> + case ets:lookup(group_servers_by_sig, {DbName, Sig}) of [] -> - case ets:lookup(couch_temp_group_fd_by_db, DbName) of - [] -> - FileName = Root ++ "/." ++ binary_to_list(DbName) ++ "_temp", - {ok, Fd} = couch_file:open(FileName, [create, overwrite]), - Count = 0; - [{_, Fd, Count}] -> - ok - end, - ?LOG_DEBUG("Spawning new temp update process for db ~s.", [DbName]), - {ok, NewPid} = couch_view_group:start_link({slow_view, DbName, Fd, Lang, DesignOptions, MapSrc, RedSrc}), - true = ets:insert(couch_temp_group_fd_by_db, {DbName, Fd, Count + 1}), - add_to_ets(NewPid, DbName, Name), - NewPid; - [{_, ExistingPid0}] -> - ExistingPid0 - end, - {reply, {ok, Pid}, Server}; -handle_call({start_group_server, DbName, GroupId}, _From, #server{root_dir=Root}=Server) -> - case ets:lookup(group_servers_by_name, {DbName, GroupId}) of - [] -> - ?LOG_DEBUG("Spawning new group server for view group ~s in database ~s.", [GroupId, DbName]), - case (catch couch_view_group:start_link({view, Root, DbName, GroupId})) of + ?LOG_DEBUG("Spawning new group server for view group ~s in database ~s.", + [GroupId, DbName]), + case (catch couch_view_group:start_link({Root, DbName, Group})) of {ok, NewPid} -> - add_to_ets(NewPid, DbName, GroupId), + add_to_ets(NewPid, DbName, Sig), {reply, {ok, NewPid}, Server}; Error -> {reply, Error, Server} @@ -272,22 +297,22 @@ end. handle_cast({reset_indexes, DbName}, #server{root_dir=Root}=Server) -> - % shutdown all the updaters + % shutdown all the updaters and clear the files, the db got changed Names = ets:lookup(couch_groups_by_db, DbName), lists:foreach( - fun({_DbName, GroupId}) -> - ?LOG_DEBUG("Killing update process for view group ~s. in database ~s.", [GroupId, DbName]), - [{_, Pid}] = ets:lookup(group_servers_by_name, {DbName, GroupId}), + fun({_DbName, Sig}) -> + ?LOG_DEBUG("Killing update process for view group ~s. in database ~s.", [Sig, DbName]), + [{_, Pid}] = ets:lookup(group_servers_by_sig, {DbName, Sig}), exit(Pid, kill), receive {'EXIT', Pid, _} -> - delete_from_ets(Pid, DbName, GroupId) + delete_from_ets(Pid, DbName, Sig) end end, Names), delete_index_dir(Root, DbName), file:delete(Root ++ "/." ++ binary_to_list(DbName) ++ "_temp"), {noreply, Server}. -handle_info({'EXIT', FromPid, Reason}, #server{root_dir=RootDir}=Server) -> +handle_info({'EXIT', FromPid, Reason}, Server) -> case ets:lookup(couch_groups_by_updater, FromPid) of [] -> if Reason /= normal -> @@ -296,40 +321,27 @@ exit(Reason); true -> ok end; - [{_, {DbName, "_temp_" ++ _ = GroupId}}] -> - delete_from_ets(FromPid, DbName, GroupId), - [{_, Fd, Count}] = ets:lookup(couch_temp_group_fd_by_db, DbName), - case Count of - 1 -> % Last ref - couch_file:close(Fd), - file:delete(RootDir ++ "/." ++ binary_to_list(DbName) ++ "_temp"), - true = ets:delete(couch_temp_group_fd_by_db, DbName); - _ -> - true = ets:insert(couch_temp_group_fd_by_db, {DbName, Fd, Count - 1}) - end; [{_, {DbName, GroupId}}] -> delete_from_ets(FromPid, DbName, GroupId) end, {noreply, Server}. -add_to_ets(Pid, DbName, GroupId) -> - true = ets:insert(couch_groups_by_updater, {Pid, {DbName, GroupId}}), - true = ets:insert(group_servers_by_name, {{DbName, GroupId}, Pid}), - true = ets:insert(couch_groups_by_db, {DbName, GroupId}). +add_to_ets(Pid, DbName, Sig) -> + true = ets:insert(couch_groups_by_updater, {Pid, {DbName, Sig}}), + true = ets:insert(group_servers_by_sig, {{DbName, Sig}, Pid}), + true = ets:insert(couch_groups_by_db, {DbName, Sig}). -delete_from_ets(Pid, DbName, GroupId) -> +delete_from_ets(Pid, DbName, Sig) -> true = ets:delete(couch_groups_by_updater, Pid), - true = ets:delete(group_servers_by_name, {DbName, GroupId}), - true = ets:delete_object(couch_groups_by_db, {DbName, GroupId}). + true = ets:delete(group_servers_by_sig, {DbName, Sig}), + true = ets:delete_object(couch_groups_by_db, {DbName, Sig}). code_change(_OldVsn, State, _Extra) -> {ok, State}. - - delete_index_dir(RootDir, DbName) -> - nuke_dir(RootDir ++ "/." ++ binary_to_list(DbName) ++ "_design"). + nuke_dir(RootDir ++ "/." ++ ?b2l(DbName) ++ "_design"). nuke_dir(Dir) -> case file:list_dir(Dir) of Modified: couchdb/trunk/src/couchdb/couch_view_group.erl URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_view_group.erl?rev=791140&r1=791139&r2=791140&view=diff ============================================================================== --- couchdb/trunk/src/couchdb/couch_view_group.erl (original) +++ couchdb/trunk/src/couchdb/couch_view_group.erl Sat Jul 4 15:47:47 2009 @@ -14,8 +14,8 @@ -behaviour(gen_server). %% API --export([start_link/1, request_group/2]). --export([design_doc_to_view_group/1]). +-export([start_link/1, request_group/2, request_group_info/1]). +-export([open_db_group/2, open_temp_group/5, design_doc_to_view_group/1,design_root/2]). %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, @@ -47,6 +47,22 @@ throw(Error) end. +request_group_info(Pid) -> + case gen_server:call(Pid, request_group_info) of + {ok, GroupInfoList} -> + {ok, GroupInfoList}; + Error -> + throw(Error) + end. + +request_index_files(Pid) -> + case gen_server:call(Pid, request_index_files) of + {ok, Filelist} -> + {ok, Filelist}; + Error -> + throw(Error) + end. + % from template start_link(InitArgs) -> @@ -67,9 +83,7 @@ Error end. -% init differentiates between temp and design_doc views. It creates a closure -% which spawns the appropriate view_updater. (It might also spawn the first -% view_updater run.) +% init creates a closure which spawns the appropriate view_updater. init({InitArgs, ReturnPid, Ref}) -> process_flag(trap_exit, true), case prepare_group(InitArgs, false) of @@ -95,8 +109,7 @@ % view group, and the couch_view_updater, which when spawned, updates the % group and sends it back here. We employ a caching mechanism, so that between % database writes, we don't have to spawn a couch_view_updater with every view -% request. This should give us more control, and the ability to request view -% statuses eventually. +% request. % The caching mechanism: each request is submitted with a seq_id for the % database at the time it was read. We guarantee to return a view from that @@ -137,8 +150,14 @@ #group_state{waiting_list=WaitList}=State) -> {noreply, State#group_state{ waiting_list=[{From, RequestSeq}|WaitList] - }, infinity}. + }, infinity}; +handle_call(request_group_info, _From, #group_state{ + group = Group, + compactor_pid = CompactorPid + } = State) -> + GroupInfo = get_group_info(Group, CompactorPid), + {reply, {ok, GroupInfo}, State}. handle_cast({start_compact, CompactFun}, #group_state{ compactor_pid=nil, group=Group, init_args={view, RootDir, DbName, GroupId} } = State) -> @@ -154,15 +173,14 @@ handle_cast({compact_done, #group{fd=NewFd, current_seq=NewSeq} = NewGroup}, #group_state{ - group = #group{current_seq=OldSeq} = Group, - init_args = {view, RootDir, DbName, GroupId}, + group = #group{current_seq=OldSeq, sig=GroupSig} = Group, + init_args = {view, RootDir, DbName, _GroupId}, updater_pid = nil, ref_counter = RefCounter } = State) when NewSeq >= OldSeq -> ?LOG_INFO("View Group compaction complete", []), - BaseName = RootDir ++ "/." ++ ?b2l(DbName) ++ ?b2l(GroupId), - FileName = BaseName ++ ".view", - CompactName = BaseName ++".compact.view", + FileName = index_file_name(RootDir, DbName, GroupSig), + CompactName = index_file_name(compact, RootDir, DbName, GroupSig), file:delete(FileName), ok = file:rename(CompactName, FileName), @@ -224,8 +242,7 @@ waiting_list=WaitList, waiting_commit=WaitingCommit}=State) when UpPid == FromPid -> ok = couch_db:close(Db), - - if Group#group.type == view andalso not WaitingCommit -> + if not WaitingCommit -> erlang:send_after(1000, self(), delayed_commit); true -> ok end, @@ -306,12 +323,13 @@ [catch gen_server:reply(Pid, Reply) || {Pid, _} <- WaitList], State#group_state{waiting_list=[]}. -prepare_group({view, RootDir, DbName, GroupId}, ForceReset)-> - case open_db_group(DbName, GroupId) of - {ok, Db, #group{sig=Sig}=Group} -> - case open_index_file(RootDir, DbName, GroupId) of +prepare_group({RootDir, DbName, #group{sig=Sig}=Group}, ForceReset)-> + case couch_db:open(DbName, []) of + {ok, Db} -> + case open_index_file(RootDir, DbName, Sig) of {ok, Fd} -> if ForceReset -> + % this can happen if we missed a purge {ok, reset_file(Db, Fd, DbName, Group)}; true -> % 09 UPGRADE CODE @@ -321,32 +339,18 @@ % sigs match! {ok, init_group(Db, Fd, Group, HeaderInfo)}; _ -> + % this happens on a new file {ok, reset_file(Db, Fd, DbName, Group)} end end; Error -> - catch delete_index_file(RootDir, DbName, GroupId), + catch delete_index_file(RootDir, DbName, Sig), Error end; - Error -> - catch delete_index_file(RootDir, DbName, GroupId), - Error - end; -prepare_group({slow_view, DbName, Fd, Lang, DesignOptions, MapSrc, RedSrc}, _ForceReset) -> - case couch_db:open(DbName, []) of - {ok, Db} -> - View = #view{map_names=[<<"_temp">>], - id_num=0, - btree=nil, - def=MapSrc, - reduce_funs= if RedSrc==[] -> []; true -> [{<<"_temp">>, RedSrc}] end}, - {ok, init_group(Db, Fd, #group{type=slow_view, name= <<"_temp">>, db=Db, - views=[View], def_lang=Lang, design_options=DesignOptions}, nil)}; - Error -> - Error + Else -> + Else end. - get_index_header_data(#group{current_seq=Seq, purge_seq=PurgeSeq, id_btree=IdBtree,views=Views}) -> ViewStates = [couch_btree:get_state(Btree) || #view{btree=Btree} <- Views], @@ -355,14 +359,47 @@ id_btree_state=couch_btree:get_state(IdBtree), view_states=ViewStates}. +hex_sig(GroupSig) -> + couch_util:to_hex(?b2l(GroupSig)). + +design_root(RootDir, DbName) -> + RootDir ++ "/." ++ ?b2l(DbName) ++ "_design/". + +index_file_name(RootDir, DbName, GroupSig) -> + design_root(RootDir, DbName) ++ hex_sig(GroupSig) ++".view". -open_index_file(RootDir, DbName, GroupId) -> - FileName = RootDir ++ "/." ++ ?b2l(DbName) ++ ?b2l(GroupId) ++".view", +index_file_name(compact, RootDir, DbName, GroupSig) -> + design_root(RootDir, DbName) ++ hex_sig(GroupSig) ++".compact.view". + + +open_index_file(RootDir, DbName, GroupSig) -> + FileName = index_file_name(RootDir, DbName, GroupSig), case couch_file:open(FileName) of {ok, Fd} -> {ok, Fd}; {error, enoent} -> couch_file:open(FileName, [create]); Error -> Error end. + +open_temp_group(DbName, Language, DesignOptions, MapSrc, RedSrc) -> + case couch_db:open(DbName, []) of + {ok, Db} -> + View = #view{map_names=[<<"_temp">>], + id_num=0, + btree=nil, + def=MapSrc, + reduce_funs= if RedSrc==[] -> []; true -> [{<<"_temp">>, RedSrc}] end}, + + {ok, Db, #group{ + name = <<"_temp">>, + db=Db, + views=[View], + def_lang=Language, + design_options=DesignOptions, + sig = erlang:md5(term_to_binary({[View], Language, DesignOptions})) + }}; + Error -> + Error + end. open_db_group(DbName, GroupId) -> case couch_db:open(DbName, []) of @@ -378,12 +415,28 @@ Else end. +get_group_info(#group{ + fd = Fd, + sig = GroupSig, + def_lang = Lang + }, CompactorPid) -> + {ok, Size} = couch_file:bytes(Fd), + [ + {signature, ?l2b(hex_sig(GroupSig))}, + {language, Lang}, + {disk_size, Size}, + {compact_running, CompactorPid /= nil} + ]. + % maybe move to another module design_doc_to_view_group(#doc{id=Id,body={Fields}}) -> Language = proplists:get_value(<<"language">>, Fields, <<"javascript">>), {DesignOptions} = proplists:get_value(<<"options">>, Fields, {[]}), {RawViews} = proplists:get_value(<<"views">>, Fields, {[]}), - + % sort the views by name to avoid spurious signature changes + SortedRawViews = lists:sort(fun({Name1, _}, {Name2, _}) -> + Name1 >= Name2 + end, RawViews), % add the views to a dictionary object, with the map source as the key DictBySrc = lists:foldl( @@ -402,7 +455,7 @@ View#view{reduce_funs=[{Name,RedSrc}|View#view.reduce_funs]} end, dict:store(MapSrc, View2, DictBySrcAcc) - end, dict:new(), RawViews), + end, dict:new(), SortedRawViews), % number the views {Views, _N} = lists:mapfoldl( fun({_Src, View}, N) -> @@ -410,7 +463,7 @@ end, 0, dict:to_list(DictBySrc)), Group = #group{name=Id, views=Views, def_lang=Language, design_options=DesignOptions}, - Group#group{sig=erlang:md5(term_to_binary(Group))}. + Group#group{sig=erlang:md5(term_to_binary({Views, Language, DesignOptions}))}. reset_group(#group{views=Views}=Group) -> Views2 = [View#view{btree=nil} || View <- Views], @@ -423,9 +476,8 @@ ok = couch_file:write_header(Fd, {Sig, nil}), init_group(Db, Fd, reset_group(Group), nil). -delete_index_file(RootDir, DbName, GroupId) -> - file:delete(RootDir ++ "/." ++ binary_to_list(DbName) - ++ binary_to_list(GroupId) ++ ".view"). +delete_index_file(RootDir, DbName, GroupSig) -> + file:delete(index_file_name(RootDir, DbName, GroupSig)). init_group(Db, Fd, #group{views=Views}=Group, nil) -> init_group(Db, Fd, Group, Modified: couchdb/trunk/src/couchdb/couch_view_updater.erl URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_view_updater.erl?rev=791140&r1=791139&r2=791140&view=diff ============================================================================== --- couchdb/trunk/src/couchdb/couch_view_updater.erl (original) +++ couchdb/trunk/src/couchdb/couch_view_updater.erl Sat Jul 4 15:47:47 2009 @@ -106,26 +106,6 @@ [conflicts, deleted_conflicts] end, case {IncludeDesign, DocId} of - {_, GroupId} -> - % uh oh. this is the design doc with our definitions. See if - % anything in the definition changed. - case couch_db:open_doc_int(Db, DocInfo, DocOpts) of - {ok, Doc} -> - case couch_view_group:design_doc_to_view_group(Doc) of - #group{sig=Sig} -> - % The same md5 signature, keep on computing - case IncludeDesign of - true -> - {[Doc | Docs], Group, ViewKVs, DocIdViewIdKeys}; - _ -> - {Docs, Group, ViewKVs, DocIdViewIdKeys} - end; - _ -> - exit(reset) - end; - {not_found, missing} -> - exit(reset) - end; {false, <>} -> % we skip design docs {Docs, Group, ViewKVs, DocIdViewIdKeys}; _ ->