couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bbast...@apache.org
Subject [25/41] couch-mrview commit: updated refs/heads/master to 28e51f3
Date Fri, 31 Oct 2014 19:53:40 GMT
couch_mrview: fix compaction with seqs_indexed=true

This change smakes sure to compact the view indexes when using the view
changes.

Conflicts:
	src/couch_mrview_compactor.erl


Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/690d878b
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/690d878b
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/690d878b

Branch: refs/heads/master
Commit: 690d878b26ce73ce713febed8a2ece4b89ba5306
Parents: b3fef4c
Author: benoitc <benoitc@apache.org>
Authored: Mon Jan 27 15:29:10 2014 +0100
Committer: Benjamin Bastian <benjamin.bastian@gmail.com>
Committed: Thu Oct 30 13:40:31 2014 -0700

----------------------------------------------------------------------
 src/couch_mrview_compactor.erl | 84 +++++++++++++++++++++++++++++++++----
 src/couch_mrview_util.erl      | 12 +++++-
 2 files changed, 88 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/690d878b/src/couch_mrview_compactor.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_compactor.erl b/src/couch_mrview_compactor.erl
index 8b1c8f7..b6a8e78 100644
--- a/src/couch_mrview_compactor.erl
+++ b/src/couch_mrview_compactor.erl
@@ -40,6 +40,8 @@ compact(State) ->
         sig=Sig,
         update_seq=Seq,
         id_btree=IdBtree,
+        log_btree=LogBtree,
+        seq_indexed=SeqIndexed,
         views=Views
     } = State,
     erlang:put(io_priority, {view_compact, DbName, IdxName}),
@@ -57,15 +59,28 @@ compact(State) ->
 
     #mrst{
         id_btree = EmptyIdBtree,
+        log_btree = EmptyLogBtree,
         views = EmptyViews
     } = EmptyState,
 
+    TotalChanges0 = case SeqIndexed of
+        true -> NumDocIds * 2;
+        _ -> NumDocIds
+    end,
+
     TotalChanges = lists:foldl(
         fun(View, Acc) ->
             {ok, Kvs} = couch_mrview_util:get_row_count(View),
-            Acc + Kvs
+            case SeqIndexed of
+                true ->
+                    {ok, SKvs} = couch_mrview_util:get_view_changes_count(View),
+                    Acc + Kvs + SKvs * 2;
+                false ->
+                    Acc + Kvs
+            end
         end,
-        NumDocIds, Views),
+        TotalChanges0, Views),
+
     couch_task_status:add_task([
         {type, view_compaction},
         {database, DbName},
@@ -107,13 +122,25 @@ compact(State) ->
     {ok, NewIdBtree} = couch_btree:add(Bt3, lists:reverse(Uncopied)),
     FinalAcc2 = update_task(FinalAcc, length(Uncopied)),
 
+
+    {NewLogBtree, FinalAcc3} = case SeqIndexed of
+        true ->
+            compact_log(LogBtree, BufferSize,
+                        FinalAcc2#acc{kvs=[],
+                                      kvs_size=0,
+                                      btree=EmptyLogBtree});
+        _ ->
+            {nil, FinalAcc2}
+    end,
+
     {NewViews, _} = lists:mapfoldl(fun({View, EmptyView}, Acc) ->
         compact_view(View, EmptyView, BufferSize, Acc)
-    end, FinalAcc2, lists:zip(Views, EmptyViews)),
+    end, FinalAcc3, lists:zip(Views, EmptyViews)),
 
     unlink(EmptyState#mrst.fd),
     {ok, EmptyState#mrst{
         id_btree=NewIdBtree,
+        log_btree=NewLogBtree,
         views=NewViews,
         update_seq=Seq
     }}.
@@ -130,9 +157,53 @@ recompact(State) ->
             {ok, State2}
     end.
 
+compact_log(LogBtree, BufferSize, Acc0) ->
+    FoldFun = fun(KV, Acc) ->
+        #acc{btree = Bt, kvs = Kvs, kvs_size = KvsSize} = Acc,
+        KvsSize2 = KvsSize + ?term_size(KV),
+        case KvsSize2 >= BufferSize of
+            true ->
+                {ok, Bt2} = couch_btree:add(Bt, lists:reverse([KV | Kvs])),
+                Acc2 = update_task(Acc, 1 + length(Kvs)),
+                {ok, Acc2#acc{
+                    btree = Bt2, kvs = [], kvs_size = 0}};
+            _ ->
+                {ok, Acc#acc{
+                    kvs = [KV | Kvs], kvs_size = KvsSize2}}
+        end
+    end,
+
+    {ok, _, FinalAcc} = couch_btree:foldl(LogBtree, FoldFun, Acc0),
+    #acc{btree = Bt3, kvs = Uncopied} = FinalAcc,
+    {ok, NewLogBtree} = couch_btree:add(Bt3, lists:reverse(Uncopied)),
+    FinalAcc2 = update_task(FinalAcc, length(Uncopied)),
+    {NewLogBtree, FinalAcc2}.
 
 %% @spec compact_view(View, EmptyView, Retry, Acc) -> {CompactView, NewAcc}
 compact_view(#mrview{id_num=VID}=View, EmptyView, BufferSize, Acc0) ->
+
+    {NewBt, Acc1} = compact_view_btree(View#mrview.btree,
+                                       EmptyView#mrview.btree,
+                                       VID, BufferSize, Acc0),
+
+    %% are we indexing changes by sequences?
+    {NewSeqBt, NewKeyBySeqBt, FinalAcc} = case View#mrview.seq_indexed of
+        true ->
+            {SBt, Acc2} = compact_view_btree(View#mrview.seq_btree,
+                                             EmptyView#mrview.seq_btree,
+                                             VID, BufferSize, Acc1),
+            {KSBt, Acc3} = compact_view_btree(View#mrview.key_byseq_btree,
+                                              EmptyView#mrview.key_byseq_btree,
+                                              VID, BufferSize, Acc2),
+            {SBt, KSBt, Acc3};
+        _ ->
+            {nil, nil, Acc1}
+    end,
+    {EmptyView#mrview{btree=NewBt,
+                      seq_btree=NewSeqBt,
+                      key_byseq_btree=NewKeyBySeqBt}, FinalAcc}.
+
+compact_view_btree(Btree, EmptyBtree, VID, BufferSize, Acc0) ->
     Fun = fun(KV, #acc{btree = Bt, kvs = Kvs, kvs_size = KvsSize} = Acc) ->
         KvsSize2 = KvsSize + ?term_size(KV),
         if KvsSize2 >= BufferSize ->
@@ -144,13 +215,12 @@ compact_view(#mrview{id_num=VID}=View, EmptyView, BufferSize, Acc0)
->
         end
     end,
 
-    InitAcc = Acc0#acc{kvs = [], kvs_size = 0, btree = EmptyView#mrview.btree},
-    {ok, _, FinalAcc} = couch_btree:foldl(View#mrview.btree, Fun, InitAcc),
+    InitAcc = Acc0#acc{kvs = [], kvs_size = 0, btree = EmptyBtree},
+    {ok, _, FinalAcc} = couch_btree:foldl(Btree, Fun, InitAcc),
     #acc{btree = Bt3, kvs = Uncopied} = FinalAcc,
     {ok, NewBt} = couch_btree:add(Bt3, lists:reverse(Uncopied)),
     FinalAcc2 = update_task(VID, FinalAcc, length(Uncopied)),
-    {EmptyView#mrview{btree=NewBt}, FinalAcc2}.
-
+    {NewBt, FinalAcc2}.
 
 update_task(Acc, ChangesInc) ->
     update_task(null, Acc, ChangesInc).

http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/690d878b/src/couch_mrview_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index 734ad26..7c639af 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -18,6 +18,7 @@
 -export([index_file/2, compaction_file/2, open_file/1]).
 -export([delete_files/2, delete_index_file/2, delete_compaction_file/2]).
 -export([get_row_count/1, all_docs_reduce_to_count/1, reduce_to_count/1]).
+-export([get_view_changes_count/1]).
 -export([all_docs_key_opts/1, all_docs_key_opts/2, key_opts/1, key_opts/2]).
 -export([fold/4, fold_reduce/4]).
 -export([temp_view_to_ddoc/1]).
@@ -333,6 +334,12 @@ reduce_to_count(Reductions) ->
     {Count, _} = couch_btree:final_reduce(Reduce, Reductions),
     Count.
 
+%% @doc get all changes for a view
+get_view_changes_count(#mrview{seq_btree=Btree}) ->
+    couch_btree:fold_reduce(
+            Btree, fun(_SeqStart, PartialReds, 0) ->
+                    {ok, couch_btree:final_reduce(Btree, PartialReds)}
+            end,0, []).
 
 fold(#mrview{btree=Bt}, Fun, Acc, Opts) ->
     WrapperFun = fun(KV, Reds, Acc2) ->
@@ -634,10 +641,13 @@ reset_state(State) ->
     State#mrst{
         fd=nil,
         qserver=nil,
+        seq_indexed=State#mrst.seq_indexed,
         update_seq=0,
         id_btree=nil,
         log_btree=nil,
-        views=[View#mrview{btree=nil, seq_btree=nil, key_byseq_btree=nil}
+        views=[View#mrview{btree=nil, seq_btree=nil,
+                           key_byseq_btree=nil,
+                           seq_indexed=View#mrview.seq_indexed}
                || View <- State#mrst.views]
     }.
 


Mime
View raw message