couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dav...@apache.org
Subject [08/25] couch-mrview commit: updated refs/heads/1994-merge-rcouch to 7775266
Date Wed, 12 Feb 2014 23:29:03 GMT
couch_mrview: fix compaction with seqs_indexed=true

This change smakes sure to compact the view indexes when using the view
changes.


Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/3d27c6be
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/3d27c6be
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/3d27c6be

Branch: refs/heads/1994-merge-rcouch
Commit: 3d27c6be52aada0bfd16b484e820e6df49626ca7
Parents: 188b30f
Author: benoitc <benoitc@apache.org>
Authored: Mon Jan 27 15:29:10 2014 +0100
Committer: Paul J. Davis <paul.joseph.davis@gmail.com>
Committed: Wed Feb 12 17:27:39 2014 -0600

----------------------------------------------------------------------
 src/couch_mrview_compactor.erl | 84 +++++++++++++++++++++++++++++++++----
 src/couch_mrview_util.erl      | 12 +++++-
 2 files changed, 88 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/3d27c6be/src/couch_mrview_compactor.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_compactor.erl b/src/couch_mrview_compactor.erl
index 4ab8dd1..a7e594f 100644
--- a/src/couch_mrview_compactor.erl
+++ b/src/couch_mrview_compactor.erl
@@ -40,6 +40,8 @@ compact(State) ->
         sig=Sig,
         update_seq=Seq,
         id_btree=IdBtree,
+        log_btree=LogBtree,
+        seq_indexed=SeqIndexed,
         views=Views
     } = State,
 
@@ -56,15 +58,28 @@ compact(State) ->
 
     #mrst{
         id_btree = EmptyIdBtree,
+        log_btree = EmptyLogBtree,
         views = EmptyViews
     } = EmptyState,
 
+    TotalChanges0 = case SeqIndexed of
+        true -> NumDocIds * 2;
+        _ -> NumDocIds
+    end,
+
     TotalChanges = lists:foldl(
         fun(View, Acc) ->
             {ok, Kvs} = couch_mrview_util:get_row_count(View),
-            Acc + Kvs
+            case SeqIndexed of
+                true ->
+                    {ok, SKvs} = couch_mrview_util:get_view_changes_count(View),
+                    Acc + Kvs + SKvs * 2;
+                false ->
+                    Acc + Kvs
+            end
         end,
-        NumDocIds, Views),
+        TotalChanges0, Views),
+
     couch_task_status:add_task([
         {type, view_compaction},
         {database, DbName},
@@ -106,13 +121,25 @@ compact(State) ->
     {ok, NewIdBtree} = couch_btree:add(Bt3, lists:reverse(Uncopied)),
     FinalAcc2 = update_task(FinalAcc, length(Uncopied)),
 
+
+    {NewLogBtree, FinalAcc3} = case SeqIndexed of
+        true ->
+            compact_log(LogBtree, BufferSize,
+                        FinalAcc2#acc{kvs=[],
+                                      kvs_size=0,
+                                      btree=EmptyLogBtree});
+        _ ->
+            {nil, FinalAcc2}
+    end,
+
     {NewViews, _} = lists:mapfoldl(fun({View, EmptyView}, Acc) ->
         compact_view(View, EmptyView, BufferSize, Acc)
-    end, FinalAcc2, lists:zip(Views, EmptyViews)),
+    end, FinalAcc3, lists:zip(Views, EmptyViews)),
 
     unlink(EmptyState#mrst.fd),
     {ok, EmptyState#mrst{
         id_btree=NewIdBtree,
+        log_btree=NewLogBtree,
         views=NewViews,
         update_seq=Seq
     }}.
@@ -129,9 +156,53 @@ recompact(State) ->
             {ok, State2}
     end.
 
+compact_log(LogBtree, BufferSize, Acc0) ->
+    FoldFun = fun({DocId, _} = KV, Acc) ->
+        #acc{btree = Bt, kvs = Kvs, kvs_size = KvsSize} = Acc,
+        KvsSize2 = KvsSize + ?term_size(KV),
+        case KvsSize2 >= BufferSize of
+            true ->
+                {ok, Bt2} = couch_btree:add(Bt, lists:reverse([KV | Kvs])),
+                Acc2 = update_task(Acc, 1 + length(Kvs)),
+                {ok, Acc2#acc{
+                    btree = Bt2, kvs = [], kvs_size = 0}};
+            _ ->
+                {ok, Acc#acc{
+                    kvs = [KV | Kvs], kvs_size = KvsSize2}}
+        end
+    end,
+
+    {ok, _, FinalAcc} = couch_btree:foldl(LogBtree, FoldFun, Acc0),
+    #acc{btree = Bt3, kvs = Uncopied} = FinalAcc,
+    {ok, NewLogBtree} = couch_btree:add(Bt3, lists:reverse(Uncopied)),
+    FinalAcc2 = update_task(FinalAcc, length(Uncopied)),
+    {NewLogBtree, FinalAcc2}.
 
 %% @spec compact_view(View, EmptyView, Retry, Acc) -> {CompactView, NewAcc}
 compact_view(View, EmptyView, BufferSize, Acc0) ->
+
+    {NewBt, Acc1} = compact_view_btree(View#mrview.btree,
+                                       EmptyView#mrview.btree,
+                                       BufferSize, Acc0),
+
+    %% are we indexing changes by sequences?
+    {NewSeqBt, NewKeyBySeqBt, FinalAcc} = case View#mrview.seq_indexed of
+        true ->
+            {SBt, Acc2} = compact_view_btree(View#mrview.seq_btree,
+                                             EmptyView#mrview.seq_btree,
+                                             BufferSize, Acc1),
+            {KSBt, Acc3} = compact_view_btree(View#mrview.key_byseq_btree,
+                                              EmptyView#mrview.key_byseq_btree,
+                                              BufferSize, Acc2),
+            {SBt, KSBt, Acc3};
+        _ ->
+            {nil, nil, Acc1}
+    end,
+    {EmptyView#mrview{btree=NewBt,
+                      seq_btree=NewSeqBt,
+                      key_byseq_btree=NewKeyBySeqBt}, FinalAcc}.
+
+compact_view_btree(Btree, EmptyBtree, BufferSize, Acc0) ->
     Fun = fun(KV, #acc{btree = Bt, kvs = Kvs, kvs_size = KvsSize} = Acc) ->
         KvsSize2 = KvsSize + ?term_size(KV),
         if KvsSize2 >= BufferSize ->
@@ -143,13 +214,12 @@ compact_view(View, EmptyView, BufferSize, Acc0) ->
         end
     end,
 
-    InitAcc = Acc0#acc{kvs = [], kvs_size = 0, btree = EmptyView#mrview.btree},
-    {ok, _, FinalAcc} = couch_btree:foldl(View#mrview.btree, Fun, InitAcc),
+    InitAcc = Acc0#acc{kvs = [], kvs_size = 0, btree = EmptyBtree},
+    {ok, _, FinalAcc} = couch_btree:foldl(Btree, Fun, InitAcc),
     #acc{btree = Bt3, kvs = Uncopied} = FinalAcc,
     {ok, NewBt} = couch_btree:add(Bt3, lists:reverse(Uncopied)),
     FinalAcc2 = update_task(FinalAcc, length(Uncopied)),
-    {EmptyView#mrview{btree=NewBt}, FinalAcc2}.
-
+    {NewBt, FinalAcc2}.
 
 update_task(#acc{changes = Changes, total_changes = Total} = Acc, ChangesInc) ->
     Changes2 = Changes + ChangesInc,

http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/3d27c6be/src/couch_mrview_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index 8cb4fae..7c42de8 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -18,6 +18,7 @@
 -export([index_file/2, compaction_file/2, open_file/1]).
 -export([delete_files/2, delete_index_file/2, delete_compaction_file/2]).
 -export([get_row_count/1, all_docs_reduce_to_count/1, reduce_to_count/1]).
+-export([get_view_changes_count/1]).
 -export([all_docs_key_opts/1, all_docs_key_opts/2, key_opts/1, key_opts/2]).
 -export([fold/4, fold_reduce/4]).
 -export([temp_view_to_ddoc/1]).
@@ -330,6 +331,12 @@ reduce_to_count(Reductions) ->
     {Count, _} = couch_btree:final_reduce(Reduce, Reductions),
     Count.
 
+%% @doc get all changes for a view
+get_view_changes_count(#mrview{seq_btree=Btree}) ->
+    couch_btree:fold_reduce(
+            Btree, fun(_SeqStart, PartialReds, 0) ->
+                    {ok, couch_btree:final_reduce(Btree, PartialReds)}
+            end,0, []).
 
 fold(#mrview{btree=Bt}, Fun, Acc, Opts) ->
     WrapperFun = fun(KV, Reds, Acc2) ->
@@ -625,10 +632,13 @@ reset_state(State) ->
     State#mrst{
         fd=nil,
         qserver=nil,
+        seq_indexed=State#mrst.seq_indexed,
         update_seq=0,
         id_btree=nil,
         log_btree=nil,
-        views=[View#mrview{btree=nil, seq_btree=nil, key_byseq_btree=nil}
+        views=[View#mrview{btree=nil, seq_btree=nil,
+                           key_byseq_btree=nil,
+                           seq_indexed=View#mrview.seq_indexed}
                || View <- State#mrst.views]
     }.
 


Mime
View raw message