couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dav...@apache.org
Subject [38/49] git commit: Improve batching of writes in view updater
Date Fri, 17 Jan 2014 22:10:23 GMT
Improve batching of writes in view updater

This patch causes the view_updater to wait until either a minimum
number of items or a maximum process memory size has been reached
before committing the changes to disk.

See COUCHDB-700 for discussion

BugzID: 11885


Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/3abc1694
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/3abc1694
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/3abc1694

Branch: refs/heads/import
Commit: 3abc16947bfa35e41db2846686b1dbf7c01a0ff1
Parents: 9f91789
Author: Adam Kocoloski <adam@cloudant.com>
Authored: Mon Aug 9 10:18:11 2010 -0400
Committer: Paul J. Davis <paul.joseph.davis@gmail.com>
Committed: Wed Mar 20 06:02:46 2013 -0500

----------------------------------------------------------------------
 src/couch_mrview_updater.erl | 44 ++++++++++++++++++++++++++++++++-------
 1 file changed, 37 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/3abc1694/src/couch_mrview_updater.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index 178cf3d..17b0432 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -167,15 +167,17 @@ compute_map_results(#mrst{qserver = Qs}, Dequeued) ->
 
 
 write_results(Parent, State) ->
-    case couch_work_queue:dequeue(State#mrst.write_queue) of
-        closed ->
+    case accumulate_writes(State, State#mrst.write_queue, nil) of
+        stop ->
             Parent ! {new_state, State};
-        {ok, Info} ->
-            EmptyKVs = [{V#mrview.id_num, []} || V <- State#mrst.views],
-            {Seq, ViewKVs, DocIdKeys} = merge_results(Info, 0, EmptyKVs, []),
+        {Go, {Seq, ViewKVs, DocIdKeys}} ->
             NewState = write_kvs(State, Seq, ViewKVs, DocIdKeys),
-            send_partial(NewState#mrst.partial_resp_pid, NewState),
-            write_results(Parent, NewState)
+            if Go == stop ->
+                Parent ! {new_state, NewState};
+            true ->
+                send_partial(NewState#mrst.partial_resp_pid, NewState),
+                write_results(Parent, NewState)
+            end
     end.
 
 
@@ -190,6 +192,34 @@ start_query_server(State) ->
     State#mrst{qserver=QServer}.
 
 
+accumulate_writes(State, W, Acc0) ->
+    {Seq, ViewKVs, DocIdKVs} = case Acc0 of
+        nil -> {0, [{V#mrview.id_num, []} || V <- State#mrst.views], []};
+        _ -> Acc0
+    end,
+    case couch_work_queue:dequeue(W) of
+        closed when Seq == 0 ->
+            stop;
+        closed ->
+            {stop, {Seq, ViewKVs, DocIdKVs}};
+        {ok, Info} ->
+            {_, _, NewIds} = Acc = merge_results(Info, Seq, ViewKVs, DocIdKVs),
+            case accumulate_more(length(NewIds)) of
+                true -> accumulate_writes(State, W, Acc);
+                false -> {ok, Acc}
+            end
+    end.
+
+
+accumulate_more(NumDocIds) ->
+    % check if we have enough items now
+    MinItems = couch_config:get("view_updater", "min_writer_items", "100"),
+    MinSize = couch_config:get("view_updater", "min_writer_size", "16777216"),
+    {memory, CurrMem} = process_info(self(), memory),
+    NumDocIds < list_to_integer(MinItems)
+        andalso CurrMem < list_to_integer(MinSize).
+
+
 merge_results([], SeqAcc, ViewKVs, DocIdKeys) ->
     {SeqAcc, ViewKVs, DocIdKeys};
 merge_results([{Seq, Results} | Rest], SeqAcc, ViewKVs, DocIdKeys) ->


Mime
View raw message