couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From fdman...@apache.org
Subject [2/3] git commit: Shutdown view group on ddoc update
Date Tue, 15 Nov 2011 13:02:31 GMT
Shutdown view group on ddoc update

If a design document is updated (or deleted), its associated
view groups that are open will be shutdown as soon as they
finish serving all their requests. Not doing this prevented
a proper cleanup of outdated view files (file descriptor leaks)
and unncessary processes in the system (old view groups and index
file ref counters, for e.g.).

COUCHDB-1309


Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/0e122004
Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/0e122004
Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/0e122004

Branch: refs/heads/1.2.x
Commit: 0e1220044f8f1dc56677cbeaa0c6cbc6fd669fe7
Parents: db99d4e
Author: Filipe David Borba Manana <fdmanana@apache.org>
Authored: Sun Nov 13 17:15:41 2011 +0000
Committer: Filipe David Borba Manana <fdmanana@apache.org>
Committed: Tue Nov 15 13:01:30 2011 +0000

----------------------------------------------------------------------
 src/couchdb/couch_view.erl             |   45 ++++++++++-----
 src/couchdb/couch_view_group.erl       |   41 ++++++++++++-
 test/etap/200-view-group-no-db-leaks.t |   83 ++++++++++++++++++++++-----
 3 files changed, 136 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb/blob/0e122004/src/couchdb/couch_view.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_view.erl b/src/couchdb/couch_view.erl
index 4c27b99..6664388 100644
--- a/src/couchdb/couch_view.erl
+++ b/src/couchdb/couch_view.erl
@@ -276,17 +276,30 @@ init([]) ->
             exit(Self, config_change)
         end),
 
+    ets:new(couch_groups_by_db, [bag, protected, named_table]),
+    ets:new(group_servers_by_sig, [set, protected, named_table]),
+    ets:new(couch_groups_by_updater, [set, private, named_table]),
+
     couch_db_update_notifier:start_link(
         fun({deleted, DbName}) ->
             gen_server:cast(couch_view, {reset_indexes, DbName});
         ({created, DbName}) ->
             gen_server:cast(couch_view, {reset_indexes, DbName});
+        ({ddoc_updated, {DbName, DDocId}}) ->
+            case ets:match_object(couch_groups_by_db, {DbName, {DDocId, '$1'}}) of
+            [] ->
+                ok;
+            [{DbName, {DDocId, Sig}}] ->
+                case ets:lookup(group_servers_by_sig, {DbName, Sig}) of
+                [{_, GroupPid}] ->
+                    (catch gen_server:cast(GroupPid, ddoc_updated));
+                [] ->
+                    ok
+                end
+            end;
         (_Else) ->
             ok
         end),
-    ets:new(couch_groups_by_db, [bag, private, named_table]),
-    ets:new(group_servers_by_sig, [set, protected, named_table]),
-    ets:new(couch_groups_by_updater, [set, private, named_table]),
     process_flag(trap_exit, true),
     ok = couch_file:init_delete_dir(RootDir),
     {ok, #server{root_dir=RootDir}}.
@@ -326,23 +339,23 @@ new_group(Root, DbName, #group{name=GroupId, sig=Sig} = Group) ->
     case (catch couch_view_group:start_link({Root, DbName, Group})) of
     {ok, NewPid} ->
         unlink(NewPid),
-        exit({DbName, Sig, {ok, NewPid}});
+        exit({DbName, GroupId, Sig, {ok, NewPid}});
     {error, invalid_view_seq} ->
         ok = gen_server:call(couch_view, {reset_indexes, DbName}),
         new_group(Root, DbName, Group);
     Error ->
-        exit({DbName, Sig, Error})
+        exit({DbName, GroupId, Sig, Error})
     end.
 
 do_reset_indexes(DbName, Root) ->
     % shutdown all the updaters and clear the files, the db got changed
     Names = ets:lookup(couch_groups_by_db, DbName),
     lists:foreach(
-        fun({_DbName, Sig}) ->
+        fun({_DbName, {DDocId, Sig}}) ->
             ?LOG_DEBUG("Killing update process for view group ~s. in database ~s.", [Sig,
DbName]),
             [{_, Pid}] = ets:lookup(group_servers_by_sig, {DbName, Sig}),
             couch_util:shutdown_sync(Pid),
-            delete_from_ets(Pid, DbName, Sig)
+            delete_from_ets(Pid, DbName, DDocId, Sig)
         end, Names),
     delete_index_dir(Root, DbName),
     RootDelDir = couch_config:get("couchdb", "view_index_dir"),
@@ -357,29 +370,31 @@ handle_info({'EXIT', FromPid, Reason}, Server) ->
             exit(Reason);
         true -> ok
         end;
-    [{_, {DbName, GroupId}}] ->
-        delete_from_ets(FromPid, DbName, GroupId)
+    [{_, {DbName, Sig}}] ->
+        [{DbName, {DDocId, Sig}}] = ets:match_object(
+            couch_groups_by_db, {DbName, {'$1', Sig}}),
+        delete_from_ets(FromPid, DbName, DDocId, Sig)
     end,
     {noreply, Server};
 
-handle_info({'DOWN', _, _, _, {DbName, Sig, Reply}}, Server) ->
+handle_info({'DOWN', _, _, _, {DbName, DDocId, Sig, Reply}}, Server) ->
     [{_, WaitList}] = ets:lookup(group_servers_by_sig, {DbName, Sig}),
     [gen_server:reply(From, Reply) || From <- WaitList],
     case Reply of {ok, NewPid} ->
         link(NewPid),
-        add_to_ets(NewPid, DbName, Sig);
+        add_to_ets(NewPid, DbName, DDocId, Sig);
      _ -> ok end,
     {noreply, Server}.
 
-add_to_ets(Pid, DbName, Sig) ->
+add_to_ets(Pid, DbName, DDocId, Sig) ->
     true = ets:insert(couch_groups_by_updater, {Pid, {DbName, Sig}}),
     true = ets:insert(group_servers_by_sig, {{DbName, Sig}, Pid}),
-    true = ets:insert(couch_groups_by_db, {DbName, Sig}).
+    true = ets:insert(couch_groups_by_db, {DbName, {DDocId, Sig}}).
 
-delete_from_ets(Pid, DbName, Sig) ->
+delete_from_ets(Pid, DbName, DDocId, Sig) ->
     true = ets:delete(couch_groups_by_updater, Pid),
     true = ets:delete(group_servers_by_sig, {DbName, Sig}),
-    true = ets:delete_object(couch_groups_by_db, {DbName, Sig}).
+    true = ets:delete_object(couch_groups_by_db, {DbName, {DDocId, Sig}}).
 
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.

http://git-wip-us.apache.org/repos/asf/couchdb/blob/0e122004/src/couchdb/couch_view_group.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_view_group.erl b/src/couchdb/couch_view_group.erl
index 6ea4e7d..62bc5af 100644
--- a/src/couchdb/couch_view_group.erl
+++ b/src/couchdb/couch_view_group.erl
@@ -35,7 +35,8 @@
     compactor_pid=nil,
     waiting_commit=false,
     waiting_list=[],
-    ref_counter=nil
+    ref_counter=nil,
+    shutdown=false
 }).
 
 % api methods
@@ -260,7 +261,33 @@ handle_cast({partial_update, Pid, NewGroup}, #group_state{updater_pid=Pid}
     end;
 handle_cast({partial_update, _, _}, State) ->
     %% message from an old (probably pre-compaction) updater; ignore
-    {noreply, State}.
+    {noreply, State};
+handle_cast(ddoc_updated, State) ->
+    #group_state{
+        db_name = DbName,
+        waiting_list = Waiters,
+        group = #group{name = DDocId, sig = CurSig}
+    } = State,
+    {ok, Db} = couch_db:open_int(DbName, []),
+    case couch_db:open_doc(Db, DDocId, [ejson_body]) of
+    {not_found, deleted} ->
+        NewSig = nil;
+    {ok, DDoc} ->
+        #group{sig = NewSig} = design_doc_to_view_group(DDoc)
+    end,
+    couch_db:close(Db),
+    case NewSig of
+    CurSig ->
+        {noreply, State#group_state{shutdown = false}};
+    _ ->
+        case Waiters of
+        [] ->
+            {stop, normal, State};
+        _ ->
+            {noreply, State#group_state{shutdown = true}}
+        end
+    end.
+
 
 handle_info(delayed_commit, #group_state{db_name=DbName,group=Group}=State) ->
     {ok, Db} = couch_db:open_int(DbName, []),
@@ -286,6 +313,7 @@ handle_info({'EXIT', FromPid, {new_group, Group}},
             updater_pid=UpPid,
             ref_counter=RefCounter,
             waiting_list=WaitList,
+            shutdown=Shutdown,
             waiting_commit=WaitingCommit}=State) when UpPid == FromPid ->
     if not WaitingCommit ->
         erlang:send_after(1000, self(), delayed_commit);
@@ -293,8 +321,13 @@ handle_info({'EXIT', FromPid, {new_group, Group}},
     end,
     case reply_with_group(Group, WaitList, [], RefCounter) of
     [] ->
-        {noreply, State#group_state{waiting_commit=true, waiting_list=[],
-                group=Group, updater_pid=nil}};
+        case Shutdown of
+        true ->
+            {stop, normal, State};
+        false ->
+            {noreply, State#group_state{waiting_commit=true, waiting_list=[],
+                group=Group, updater_pid=nil}}
+        end;
     StillWaiting ->
         % we still have some waiters, reopen the database and reupdate the index
         Owner = self(),

http://git-wip-us.apache.org/repos/asf/couchdb/blob/0e122004/test/etap/200-view-group-no-db-leaks.t
----------------------------------------------------------------------
diff --git a/test/etap/200-view-group-no-db-leaks.t b/test/etap/200-view-group-no-db-leaks.t
index f17b6ae..936e0a9 100755
--- a/test/etap/200-view-group-no-db-leaks.t
+++ b/test/etap/200-view-group-no-db-leaks.t
@@ -52,7 +52,7 @@ ddoc_name() -> <<"foo">>.
 main(_) ->
     test_util:init_code_path(),
 
-    etap:plan(18),
+    etap:plan(28),
     case (catch test()) of
         ok ->
             etap:end_tests();
@@ -72,19 +72,19 @@ test() ->
     create_db(),
 
     create_docs(),
-    create_design_doc(),
+    {ok, DDocRev} = create_design_doc(),
 
     ViewGroup = couch_view:get_group_server(
         test_db_name(), <<"_design/", (ddoc_name())/binary>>),
     etap:is(is_pid(ViewGroup), true, "got view group pid"),
     etap:is(is_process_alive(ViewGroup), true, "view group pid is alive"),
 
-    query_view(),
+    query_view(3, null, false),
     check_db_ref_count(),
     etap:is(is_process_alive(ViewGroup), true, "view group pid is alive"),
 
     create_new_doc(<<"doc1000">>),
-    query_view(),
+    query_view(4, null, false),
     check_db_ref_count(),
     etap:is(is_process_alive(ViewGroup), true, "view group pid is alive"),
 
@@ -103,17 +103,38 @@ test() ->
     etap:is(is_process_alive(ViewGroup), true, "view group pid is alive"),
 
     create_new_doc(<<"doc1001">>),
-    query_view(),
+    query_view(5, null, false),
     check_db_ref_count(),
     etap:is(is_process_alive(ViewGroup), true, "view group pid is alive"),
 
+    etap:diag("updating the design document with a new view definition"),
+    {ok, _NewDDocRev} = update_ddoc_view(DDocRev),
+
+    NewViewGroup = couch_view:get_group_server(
+        test_db_name(), <<"_design/", (ddoc_name())/binary>>),
+    etap:is(is_pid(NewViewGroup), true, "got new view group pid"),
+    etap:is(is_process_alive(NewViewGroup), true, "new view group pid is alive"),
+    etap:isnt(NewViewGroup, ViewGroup, "new view group has a different pid"),
+    etap:diag("querying view with ?stale=ok, must return empty row set"),
+    query_view(0, foo, ok),
+    etap:diag("querying view (without stale), must return 5 rows with value 1"),
+    query_view(5, 1, false),
     MonRef = erlang:monitor(process, ViewGroup),
-    ok = couch_server:delete(test_db_name(), []),
     receive
     {'DOWN', MonRef, _, _, _} ->
-        etap:diag("view group is dead after DB deletion")
+        etap:diag("old view group is dead after ddoc update")
     after 5000 ->
-        etap:bail("view group did not die after DB deletion")
+        etap:bail("old view group is not dead after ddoc update")
+    end,
+
+    etap:diag("deleting database"),
+    MonRef2 = erlang:monitor(process, NewViewGroup),
+    ok = couch_server:delete(test_db_name(), []),
+    receive
+    {'DOWN', MonRef2, _, _, _} ->
+        etap:diag("new view group is dead after DB deletion")
+    after 5000 ->
+        etap:bail("new view group did not die after DB deletion")
     end,
 
     ok = timer:sleep(1000),
@@ -220,9 +241,27 @@ create_design_doc() ->
             ]}}
         ]}}
     ]}),
-    {ok, _} = couch_db:update_docs(Db, [DDoc]),
+    {ok, Rev} = couch_db:update_doc(Db, DDoc, []),
     couch_db:ensure_full_commit(Db),
-    couch_db:close(Db).
+    couch_db:close(Db),
+    {ok, Rev}.
+
+update_ddoc_view(DDocRev) ->
+    {ok, Db} = couch_db:open(test_db_name(), [admin_user_ctx()]),
+    DDoc = couch_doc:from_json_obj({[
+        {<<"_id">>, <<"_design/", (ddoc_name())/binary>>},
+        {<<"_rev">>, couch_doc:rev_to_str(DDocRev)},
+        {<<"language">>, <<"javascript">>},
+        {<<"views">>, {[
+            {<<"bar">>, {[
+                {<<"map">>, <<"function(doc) { emit(doc._id, 1); }">>}
+            ]}}
+        ]}}
+    ]}),
+    {ok, NewRev} = couch_db:update_doc(Db, DDoc, []),
+    couch_db:ensure_full_commit(Db),
+    couch_db:close(Db),
+    {ok, NewRev}.
 
 create_new_doc(Id) ->
     {ok, Db} = couch_db:open(test_db_name(), [admin_user_ctx()]),
@@ -238,10 +277,26 @@ db_url() ->
     "http://" ++ get(addr) ++ ":" ++ get(port) ++ "/" ++
     binary_to_list(test_db_name()).
 
-query_view() ->
-    {ok, Code, _Headers, _Body} = test_util:request(
-        db_url() ++ "/_design/" ++ binary_to_list(ddoc_name()) ++ "/_view/bar",
+query_view(ExpectedRowCount, ExpectedRowValue, Stale) ->
+    {ok, Code, _Headers, Body} = test_util:request(
+        db_url() ++ "/_design/" ++ binary_to_list(ddoc_name()) ++ "/_view/bar"
+          ++ case Stale of
+                 false -> [];
+                 _ -> "?stale=" ++ atom_to_list(Stale)
+             end,
         [],
         get),
     etap:is(Code, 200, "got view response"),
-    ok.
+    {Props} = ejson:decode(Body),
+    Rows = couch_util:get_value(<<"rows">>, Props, []),
+    etap:is(length(Rows), ExpectedRowCount, "result set has correct # of rows"),
+    lists:foreach(
+        fun({Row}) ->
+            case couch_util:get_value(<<"value">>, Row) of
+            ExpectedRowValue ->
+                ok;
+            _ ->
+                etap:bail("row has incorrect value")
+            end
+        end,
+        Rows).


Mime
View raw message