couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e...@apache.org
Subject couch-mrview commit: updated refs/heads/master to 6ec3597
Date Tue, 19 Jan 2016 15:51:23 GMT
Repository: couchdb-couch-mrview
Updated Branches:
  refs/heads/master 2341c38c6 -> 6ec35971e


Refactor to add an interface for view_state of #mrheader

The goal of this refactoring is to make it easier to understand
and maintain view_state data structure of #mrheader.

Unlike record #mrview in #mrst, the data structure that represents
view_states in #mrheader is a tuple (of tuples) and its definition
and application spreads across all couch_mrview_util.erl module.
This makes it hard to follow and understand how it used
or troubleshoot mrview in case of the problems.

This change introduces an interface for generation and accessing
view_state and also separates generation of reduction closures
from mrview's `reduce`, `reduce_to_count` and `fold_reduce` functions
as they rely on view_state's structure.

Note that this refactoring also  removes explicit set of generic
ordering function on collation option "raw" as it is a default
of couch_btree:open on missing option for `less` function.


Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/6ec35971
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/6ec35971
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/6ec35971

Branch: refs/heads/master
Commit: 6ec35971ebd96133cc6a34fa29f1d364049a7ecd
Parents: 2341c38
Author: Eric Avdey <eiri@eiri.ca>
Authored: Fri Jan 15 16:12:55 2016 -0400
Committer: Eric Avdey <eiri@eiri.ca>
Committed: Tue Jan 19 11:39:40 2016 -0400

----------------------------------------------------------------------
 src/couch_mrview_util.erl | 235 +++++++++++++++++++++++++----------------
 1 file changed, 143 insertions(+), 92 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/6ec35971/src/couch_mrview_util.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index c30460c..51e51d8 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -200,7 +200,7 @@ init_state(Db, Fd, #mrst{views=Views}=State, nil) ->
         purge_seq=couch_db:get_purge_seq(Db),
         id_btree_state=nil,
         log_btree_state=nil,
-        view_states=[{nil, nil, nil, 0, 0} || _ <- Views]
+        view_states=[make_view_state(#mrview{}) || _ <- Views]
     },
     init_state(Db, Fd, State, Header);
 % read <= 1.2.x header record and transpile it to >=1.3.x
@@ -215,8 +215,8 @@ init_state(Db, Fd, State, #index_header{
         purge_seq=PurgeSeq,
         id_btree_state=IdBtreeState,
         log_btree_state=nil,
-        view_states=[{Bt, nil, nil, USeq, PSeq} || {Bt, USeq, PSeq} <- ViewStates]
-        });
+        view_states=[make_view_state(V) || V <- ViewStates]
+    });
 init_state(Db, Fd, State, Header) ->
     #mrst{
         language=Lang,
@@ -232,12 +232,6 @@ init_state(Db, Fd, State, Header) ->
         view_states=ViewStates
     } = Header,
 
-    StateUpdate = fun
-        ({_, _, _, _, _}=St) -> St;
-        (St) -> {St, nil, nil, 0, 0}
-    end,
-    ViewStates2 = lists:map(StateUpdate, ViewStates),
-
     IdBtOpts = [{compression, couch_db:compression(Db)}],
     {ok, IdBtree} = couch_btree:open(IdBtreeState, Fd, IdBtOpts),
     {ok, LogBtree} = case SeqIndexed orelse KeySeqIndexed of
@@ -246,7 +240,7 @@ init_state(Db, Fd, State, Header) ->
     end,
 
     OpenViewFun = fun(St, View) -> open_view(Db, Fd, Lang, St, View) end,
-    Views2 = lists:zipwith(OpenViewFun, ViewStates2, Views),
+    Views2 = lists:zipwith(OpenViewFun, ViewStates, Views),
 
     State#mrst{
         fd=Fd,
@@ -258,45 +252,33 @@ init_state(Db, Fd, State, Header) ->
         views=Views2
     }.
 
-open_view(Db, Fd, Lang, {BTState, SeqBTState, KSeqBTState, USeq, PSeq}, View) ->
-    FunSrcs = [FunSrc || {_Name, FunSrc} <- View#mrview.reduce_funs],
-    ReduceFun =
-        fun(reduce, KVs) ->
-            KVs2 = detuple_kvs(expand_dups(KVs, []), []),
-            {ok, Result} = couch_query_servers:reduce(Lang, FunSrcs, KVs2),
-            {length(KVs2), Result};
-        (rereduce, Reds) ->
-            Count = lists:sum([Count0 || {Count0, _} <- Reds]),
-            UsrReds = [UsrRedsList || {_, UsrRedsList} <- Reds],
-            {ok, Result} = couch_query_servers:rereduce(Lang, FunSrcs, UsrReds),
-            {Count, Result}
-        end,
-
-    Less = case couch_util:get_value(<<"collation">>, View#mrview.options) of
-        <<"raw">> -> fun(A, B) -> A < B end;
-        _ -> fun couch_ejson_compare:less_json_ids/2
-    end,
-
+open_view(Db, Fd, Lang, ViewState, View) ->
+    ReduceFun = make_reduce_fun(Lang, View#mrview.reduce_funs),
+    LessFun = maybe_define_less_fun(View),
+    Compression = couch_db:compression(Db),
+    BTState = get_key_btree_state(ViewState),
     ViewBtOpts = [
-        {less, Less},
+        {less, LessFun},
         {reduce, ReduceFun},
-        {compression, couch_db:compression(Db)}
+        {compression, Compression}
     ],
     {ok, Btree} = couch_btree:open(BTState, Fd, ViewBtOpts),
 
     BySeqReduceFun = fun couch_db_updater:btree_by_seq_reduce/2,
     {ok, SeqBtree} = if View#mrview.seq_indexed ->
+        SeqBTState = get_seq_btree_state(ViewState),
         ViewSeqBtOpts = [{reduce, BySeqReduceFun},
-                         {compression, couch_db:compression(Db)}],
+                         {compression, Compression}],
 
         couch_btree:open(SeqBTState, Fd, ViewSeqBtOpts);
     true ->
         {ok, nil}
     end,
     {ok, KeyBySeqBtree} = if View#mrview.keyseq_indexed ->
-        KeyBySeqBtOpts = [{less, Less},
+        KSeqBTState = get_kseq_btree_state(ViewState),
+        KeyBySeqBtOpts = [{less, LessFun},
                           {reduce, BySeqReduceFun},
-                          {compression, couch_db:compression(Db)}],
+                          {compression, Compression}],
         couch_btree:open(KSeqBTState, Fd, KeyBySeqBtOpts);
     true ->
         {ok, nil}
@@ -305,8 +287,8 @@ open_view(Db, Fd, Lang, {BTState, SeqBTState, KSeqBTState, USeq, PSeq},
View) ->
     View#mrview{btree=Btree,
                 seq_btree=SeqBtree,
                 key_byseq_btree=KeyBySeqBtree,
-                update_seq=USeq,
-                purge_seq=PSeq}.
+                update_seq=get_update_seq(ViewState),
+                purge_seq=get_purge_seq(ViewState)}.
 
 
 temp_view_to_ddoc({Props}) ->
@@ -341,18 +323,9 @@ all_docs_reduce_to_count(Reductions) ->
 reduce_to_count(nil) ->
     0;
 reduce_to_count(Reductions) ->
-    Reduce = fun
-        (reduce, KVs) ->
-            Counts = [
-                case V of {dups, Vals} -> length(Vals); _ -> 1 end
-                || {_,V} <- KVs
-            ],
-            {lists:sum(Counts), []};
-        (rereduce, Reds) ->
-            {lists:sum([Count0 || {Count0, _} <- Reds]), []}
-    end,
-    {Count, _} = couch_btree:final_reduce(Reduce, Reductions),
-    Count.
+    CountReduceFun = fun count_reduce/2,
+    FinalReduction = couch_btree:final_reduce(CountReduceFun, Reductions),
+    get_count(FinalReduction).
 
 %% @doc get all changes for a view
 get_view_changes_count(View) ->
@@ -414,25 +387,13 @@ fold_reduce({NthRed, Lang, View}, Fun,  Acc, Options) ->
         btree=Bt,
         reduce_funs=RedFuns
     } = View,
-    LPad = lists:duplicate(NthRed - 1, []),
-    RPad = lists:duplicate(length(RedFuns) - NthRed, []),
-    {_Name, FunSrc} = lists:nth(NthRed,RedFuns),
 
-    ReduceFun = fun
-        (reduce, KVs0) ->
-            KVs1 = detuple_kvs(expand_dups(KVs0, []), []),
-            {ok, Red} = couch_query_servers:reduce(Lang, [FunSrc], KVs1),
-            {0, LPad ++ Red ++ RPad};
-        (rereduce, Reds) ->
-            ExtractRed = fun({_, UReds0}) -> [lists:nth(NthRed, UReds0)] end,
-            UReds = lists:map(ExtractRed, Reds),
-            {ok, Red} = couch_query_servers:rereduce(Lang, [FunSrc], UReds),
-            {0, LPad ++ Red ++ RPad}
-    end,
+    ReduceFun = make_user_reds_reduce_fun(Lang, RedFuns, NthRed),
 
     WrapperFun = fun({GroupedKey, _}, PartialReds, Acc0) ->
-        {_, Reds} = couch_btree:final_reduce(ReduceFun, PartialReds),
-        Fun(GroupedKey, lists:nth(NthRed, Reds), Acc0)
+        FinalReduction = couch_btree:final_reduce(ReduceFun, PartialReds),
+        UserReductions = get_user_reds(FinalReduction),
+        Fun(GroupedKey, lists:nth(NthRed, UserReductions), Acc0)
     end,
 
     couch_btree:fold_reduce(Bt, WrapperFun, Acc, Options).
@@ -610,37 +571,12 @@ make_header(State) ->
         views=Views
     } = State,
 
-    ViewStates = lists:foldr(fun(V, Acc) ->
-                    SeqBtState = case V#mrview.seq_indexed of
-                        true ->
-                            couch_btree:get_state(V#mrview.seq_btree);
-                        _ ->
-                            nil
-                    end,
-                    KSeqBtState = case V#mrview.keyseq_indexed of
-                        true ->
-                            couch_btree:get_state(V#mrview.key_byseq_btree);
-                        _ ->
-                            nil
-                    end,
-                    [{couch_btree:get_state(V#mrview.btree),
-                      SeqBtState,
-                      KSeqBtState,
-                      V#mrview.update_seq,
-                      V#mrview.purge_seq} | Acc]
-            end, [], Views),
-
-    LogBtreeState = case LogBtree of
-        nil -> nil;
-        _ -> couch_btree:get_state(LogBtree)
-    end,
-
     #mrheader{
         seq=Seq,
         purge_seq=PurgeSeq,
-        id_btree_state=couch_btree:get_state(IdBtree),
-        log_btree_state= LogBtreeState,
-        view_states=ViewStates
+        id_btree_state=get_btree_state(IdBtree),
+        log_btree_state=get_btree_state(LogBtree),
+        view_states=[make_view_state(V) || V <- Views]
     }.
 
 
@@ -1004,6 +940,121 @@ old_view_format(View) ->
 
 %% End of <= 1.2.x upgrade code.
 
+make_view_state(#mrview{} = View) ->
+    BTState = get_btree_state(View#mrview.btree),
+    SeqBTState = case View#mrview.seq_indexed of
+        true ->
+            get_btree_state(View#mrview.seq_btree);
+        _ ->
+            nil
+    end,
+    KSeqBTState = case View#mrview.keyseq_indexed of
+        true ->
+            get_btree_state(View#mrview.key_byseq_btree);
+        _ ->
+            nil
+    end,
+    {
+        BTState,
+        SeqBTState,
+        KSeqBTState,
+        View#mrview.update_seq,
+        View#mrview.purge_seq
+    };
+make_view_state({BTState, UpdateSeq, PurgeSeq}) ->
+    {BTState, nil, nil, UpdateSeq, PurgeSeq};
+make_view_state(nil) ->
+    {nil, nil, nil, 0, 0}.
+
+
+get_key_btree_state(ViewState) ->
+    element(1, ViewState).
+
+get_seq_btree_state(ViewState) ->
+    element(2, ViewState).
+
+get_kseq_btree_state(ViewState) ->
+    element(3, ViewState).
+
+get_update_seq(ViewState) ->
+    element(4, ViewState).
+
+get_purge_seq(ViewState) ->
+    element(5, ViewState).
+
+get_count(Reduction) ->
+    element(1, Reduction).
+
+get_user_reds(Reduction) ->
+    element(2, Reduction).
+
+
+make_reduce_fun(Lang, ReduceFuns) ->
+    FunSrcs = [FunSrc || {_, FunSrc} <- ReduceFuns],
+    fun
+        (reduce, KVs0) ->
+            KVs = detuple_kvs(expand_dups(KVs0, []), []),
+            {ok, Result} = couch_query_servers:reduce(Lang, FunSrcs, KVs),
+            {length(KVs), Result};
+        (rereduce, Reds) ->
+            ExtractFun = fun(Red, {CountsAcc0, URedsAcc0}) ->
+                CountsAcc = CountsAcc0 + get_count(Red),
+                URedsAcc = lists:append(URedsAcc0, [get_user_reds(Red)]),
+                {CountsAcc, URedsAcc}
+            end,
+            {Counts, UReds} = lists:foldl(ExtractFun, {0, []}, Reds),
+            {ok, Result} = couch_query_servers:rereduce(Lang, FunSrcs, UReds),
+            {Counts, Result}
+    end.
+
+
+maybe_define_less_fun(#mrview{options = Options}) ->
+    case couch_util:get_value(<<"collation">>, Options) of
+        <<"raw">> -> undefined;
+        _ -> fun couch_ejson_compare:less_json_ids/2
+    end.
+
+
+count_reduce(reduce, KVs) ->
+    CountFun = fun
+        ({_, {dups, Vals}}, Acc) -> Acc + length(Vals);
+        (_, Acc) -> Acc + 1
+    end,
+    Count = lists:foldl(CountFun, 0, KVs),
+    {Count, []};
+count_reduce(rereduce, Reds) ->
+    CountFun = fun(Red, Acc) ->
+        Acc + get_count(Red)
+    end,
+    Count = lists:foldl(CountFun, 0, Reds),
+    {Count, []}.
+
+
+make_user_reds_reduce_fun(Lang, ReduceFuns, NthRed) ->
+    LPad = lists:duplicate(NthRed - 1, []),
+    RPad = lists:duplicate(length(ReduceFuns) - NthRed, []),
+    {_, FunSrc} = lists:nth(NthRed, ReduceFuns),
+    fun
+        (reduce, KVs0) ->
+            KVs = detuple_kvs(expand_dups(KVs0, []), []),
+            {ok, Result} = couch_query_servers:reduce(Lang, [FunSrc], KVs),
+            {0, LPad ++ Result ++ RPad};
+        (rereduce, Reds) ->
+            ExtractFun = fun(Reds0) ->
+                [lists:nth(NthRed, get_user_reds(Reds0))]
+            end,
+            UReds = lists:map(ExtractFun, Reds),
+            {ok, Result} = couch_query_servers:rereduce(Lang, [FunSrc], UReds),
+            {0, LPad ++ Result ++ RPad}
+    end.
+
+
+get_btree_state(nil) ->
+    nil;
+get_btree_state(#btree{} = Btree) ->
+    couch_btree:get_state(Btree).
+
+
 extract_view_reduce({red, {N, _Lang, #mrview{reduce_funs=Reds}}, _Ref}) ->
     {_Name, FunSrc} = lists:nth(N, Reds),
     FunSrc.


Mime
View raw message