couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From robertkowal...@apache.org
Subject [38/50] [abbrv] couchdb-mango git commit: Move view specific logic to mango_cursor_view
Date Tue, 03 Feb 2015 15:13:45 GMT
Move view specific logic to mango_cursor_view

A good portion of mango_cursor turned out to be specific to view based
indexes. This moves the view related logic to mango_cursor_view and
updates mango_cursor to be more generic in preparation for the text
based indexing.

BugzId: 33294


Project: http://git-wip-us.apache.org/repos/asf/couchdb-mango/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mango/commit/06bf7757
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mango/tree/06bf7757
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mango/diff/06bf7757

Branch: refs/heads/master
Commit: 06bf7757b9f98c7cf5d5c597713021f6cb4ae27b
Parents: 7a9de34
Author: Paul J. Davis <paul.joseph.davis@gmail.com>
Authored: Fri Jan 9 14:21:12 2015 -0600
Committer: Paul J. Davis <paul.joseph.davis@gmail.com>
Committed: Fri Jan 16 13:32:49 2015 -0600

----------------------------------------------------------------------
 src/mango_cursor.erl      | 160 +++++++++--------------------------------
 src/mango_cursor_view.erl |  71 ++++++++++++++++++
 src/mango_error.erl       |   4 +-
 src/mango_idx.erl         |  36 ++++++++++
 src/mango_idx_special.erl |   6 ++
 src/mango_idx_view.erl    |   9 +++
 6 files changed, 156 insertions(+), 130 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06bf7757/src/mango_cursor.erl
----------------------------------------------------------------------
diff --git a/src/mango_cursor.erl b/src/mango_cursor.erl
index 8f780b7..5bfe383 100644
--- a/src/mango_cursor.erl
+++ b/src/mango_cursor.erl
@@ -29,34 +29,24 @@
 
 create(Db, Selector0, Opts) ->
     Selector = mango_selector:normalize(Selector0),
-    IndexFields = mango_idx_view:indexable_fields(Selector),    
-    FieldRanges = mango_idx_view:field_ranges(Selector, IndexFields),
-
-    if IndexFields /= [] -> ok; true ->
-        ?MANGO_ERROR({no_usable_index, operator_unsupported})
-    end,
 
     ExistingIndexes = mango_idx:list(Db),
-    UsableIndexes = find_usable_indexes(IndexFields, ExistingIndexes),
-    SortIndexes = get_sort_indexes(ExistingIndexes, UsableIndexes, Opts),
+    if ExistingIndexes /= [] -> ok; true ->
+        ?MANGO_ERROR({no_usable_index, no_indexes_defined})
+    end,
 
-    Composited = composite_indexes(SortIndexes, FieldRanges),
-    {Index, Ranges} = choose_best_index(Db, Composited),
+    SortIndexes = mango_idx:for_sort(ExistingIndexes, Opts),
+    if SortIndexes /= [] -> ok; true ->
+        ?MANGO_ERROR({no_usable_index, missing_sort_index})
+    end,
 
-    Limit = couch_util:get_value(limit, Opts, 10000000000),
-    Skip = couch_util:get_value(skip, Opts, 0),
-    Fields = couch_util:get_value(fields, Opts, all_fields),
+    UsableFilter = fun(I) -> mango_idx:is_usable(I, Selector) end,
+    UsableIndexes = lists:filter(UsableFilter, SortIndexes),
+    if UsableIndexes /= [] -> ok; true ->
+        ?MANGO_ERROR({no_usable_index, selector_unsupported})
+    end,
 
-    {ok, #cursor{
-        db = Db,
-        index = Index,
-        ranges = Ranges,
-        selector = Selector,
-        opts = Opts,
-        limit = Limit,
-        skip = Skip,
-        fields = Fields
-    }}.
+    create_cursor(Db, UsableIndexes, Selector, Opts).
 
 
 execute(#cursor{index=Idx}=Cursor, UserFun, UserAcc) ->
@@ -64,109 +54,23 @@ execute(#cursor{index=Idx}=Cursor, UserFun, UserAcc) ->
     Mod:execute(Cursor, UserFun, UserAcc).
 
 
-% Find the intersection between the Possible and Existing
-% indexes.
-find_usable_indexes([], _) ->
-    ?MANGO_ERROR({no_usable_index, query_unsupported});
-find_usable_indexes(Possible, []) ->
-    ?MANGO_ERROR({no_usable_index, {fields, Possible}});
-find_usable_indexes(Possible, Existing) ->
-    Usable = lists:foldl(fun(Idx, Acc) ->
-        [Col0 | _] = mango_idx:columns(Idx),
-        case lists:member(Col0, Possible) of
-            true ->
-                [Idx | Acc];
-            false ->
-                Acc
+create_cursor(Db, Indexes, Selector, Opts) ->
+    [{CursorMod, CursorModIndexes} | _] = group_indexes_by_type(Indexes),
+    CursorMod:create(Db, CursorModIndexes, Selector, Opts).
+
+
+group_indexes_by_type(Indexes) ->
+    IdxDict = lists:foldl(fun(I, D) ->
+        dict:append(mango_idx:cursor_mod(I), I, D)
+    end, dict:new(), Indexes),
+    CursorModules = [
+        mango_cursor_view
+    ],
+    lists:flatmap(fun(CMod) ->
+        case dict:find(CMod, IdxDict) of
+            {ok, CModIndexes} ->
+                [{CMod, CModIndexes}];
+            error ->
+                []
         end
-    end, [], Existing),
-    if length(Usable) > 0 -> ok; true ->
-        ?MANGO_ERROR({no_usable_index, {fields, Possible}})
-    end,
-    Usable.
-
-
-get_sort_indexes(ExistingIndexes, UsableIndexes, Opts) ->
-    % If a sort was specified we have to find an index that
-    % can satisfy the request.
-    case lists:keyfind(sort, 1, Opts) of
-        {sort, {[_ | _]} = Sort} ->
-            limit_to_sort(ExistingIndexes, UsableIndexes, Sort);
-        _ ->
-            UsableIndexes
-    end.
-
-
-limit_to_sort(ExistingIndexes, UsableIndexes, Sort) ->
-    Fields = mango_sort:fields(Sort),
-
-    % First make sure that we have an index that could
-    % answer this sort. We split like this so that the
-    % user error is more obvious.
-    SortFilt = fun(Idx) ->
-        Cols = mango_idx:columns(Idx),
-        lists:prefix(Fields, Cols)
-    end,
-    SortIndexes = lists:filter(SortFilt, ExistingIndexes),
-    if SortIndexes /= [] -> ok; true ->
-        ?MANGO_ERROR({no_usable_index, {sort, Fields}})
-    end,
-
-    % And then check if one or more of our SortIndexes
-    % is usable.
-    UsableFilt = fun(Idx) -> lists:member(Idx, UsableIndexes) end,
-    FinalIndexes = lists:filter(UsableFilt, SortIndexes),
-    if FinalIndexes /= [] -> ok; true ->
-        ?MANGO_ERROR({no_usable_index, sort_field})
-    end,
-
-    FinalIndexes.
-
-
-% Any of these indexes may be a composite index. For each
-% index find the most specific set of fields for each
-% index. Ie, if an index has columns a, b, c, d, then
-% check FieldRanges for a, b, c, and d and return
-% the longest prefix of columns found.
-composite_indexes(Indexes, FieldRanges) ->
-    lists:foldl(fun(Idx, Acc) ->
-        Cols = mango_idx:columns(Idx),
-        Prefix = composite_prefix(Cols, FieldRanges),
-        [{Idx, Prefix} | Acc]
-    end, [], Indexes).
-
-
-composite_prefix([], _) ->
-    [];
-composite_prefix([Col | Rest], Ranges) ->
-    case lists:keyfind(Col, 1, Ranges) of
-        {Col, Range} ->
-            [Range | composite_prefix(Rest, Ranges)];
-        false ->
-            []
-    end.
-
-
-% Low and behold our query planner. Or something.
-% So stupid, but we can fix this up later. First
-% pass: Sort the IndexRanges by (num_columns, idx_name)
-% and return the first element. Yes. Its going to
-% be that dumb for now.
-%
-% In the future we can look into doing a cached parallel
-% reduce view read on each index with the ranges to find
-% the one that has the fewest number of rows or something.
-choose_best_index(_DbName, IndexRanges) ->
-    Cmp = fun({A1, A2}, {B1, B2}) ->
-        case length(A2) - length(B2) of
-            N when N < 0 -> true;
-            N when N == 0 ->
-                % This is a really bad sort and will end
-                % up preferring indices based on the
-                % (dbname, ddocid, view_name) triple
-                A1 =< B1;
-            _ ->
-                false
-        end
-    end,
-    hd(lists:sort(Cmp, IndexRanges)).
+    end, CursorModules).

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06bf7757/src/mango_cursor_view.erl
----------------------------------------------------------------------
diff --git a/src/mango_cursor_view.erl b/src/mango_cursor_view.erl
index a9f66b0..798b713 100644
--- a/src/mango_cursor_view.erl
+++ b/src/mango_cursor_view.erl
@@ -13,6 +13,7 @@
 -module(mango_cursor_view).
 
 -export([
+    create/4,
     execute/3
 ]).
 
@@ -25,6 +26,27 @@
 -include("mango_cursor.hrl").
 
 
+create(Db, Indexes, Selector, Opts) ->
+    FieldRanges = mango_idx_view:field_ranges(Selector),
+    Composited = composite_indexes(Indexes, FieldRanges),
+    {Index, IndexRanges} = choose_best_index(Db, Composited),
+
+    Limit = couch_util:get_value(limit, Opts, 10000000000),
+    Skip = couch_util:get_value(skip, Opts, 0),
+    Fields = couch_util:get_value(fields, Opts, all_fields),
+
+    {ok, #cursor{
+        db = Db,
+        index = Index,
+        ranges = IndexRanges,
+        selector = Selector,
+        opts = Opts,
+        limit = Limit,
+        skip = Skip,
+        fields = Fields
+    }}.
+
+
 execute(#cursor{db = Db, index = Idx} = Cursor0, UserFun, UserAcc) ->
     Cursor = Cursor0#cursor{
         user_fun = UserFun,
@@ -52,6 +74,55 @@ execute(#cursor{db = Db, index = Idx} = Cursor0, UserFun, UserAcc) ->
     {ok, LastCursor#cursor.user_acc}.
 
 
+% Any of these indexes may be a composite index. For each
+% index find the most specific set of fields for each
+% index. Ie, if an index has columns a, b, c, d, then
+% check FieldRanges for a, b, c, and d and return
+% the longest prefix of columns found.
+composite_indexes(Indexes, FieldRanges) ->
+    lists:foldl(fun(Idx, Acc) ->
+        Cols = mango_idx:columns(Idx),
+        Prefix = composite_prefix(Cols, FieldRanges),
+        [{Idx, Prefix} | Acc]
+    end, [], Indexes).
+
+
+composite_prefix([], _) ->
+    [];
+composite_prefix([Col | Rest], Ranges) ->
+    case lists:keyfind(Col, 1, Ranges) of
+        {Col, Range} ->
+            [Range | composite_prefix(Rest, Ranges)];
+        false ->
+            []
+    end.
+
+
+% Low and behold our query planner. Or something.
+% So stupid, but we can fix this up later. First
+% pass: Sort the IndexRanges by (num_columns, idx_name)
+% and return the first element. Yes. Its going to
+% be that dumb for now.
+%
+% In the future we can look into doing a cached parallel
+% reduce view read on each index with the ranges to find
+% the one that has the fewest number of rows or something.
+choose_best_index(_DbName, IndexRanges) ->
+    Cmp = fun({A1, A2}, {B1, B2}) ->
+        case length(A2) - length(B2) of
+            N when N < 0 -> true;
+            N when N == 0 ->
+                % This is a really bad sort and will end
+                % up preferring indices based on the
+                % (dbname, ddocid, view_name) triple
+                A1 =< B1;
+            _ ->
+                false
+        end
+    end,
+    hd(lists:sort(Cmp, IndexRanges)).
+
+
 handle_message({total_and_offset, _, _} = _TO, Cursor) ->
     %twig:log(err, "TOTAL AND OFFSET: ~p", [_TO]),
     {ok, Cursor};

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06bf7757/src/mango_error.erl
----------------------------------------------------------------------
diff --git a/src/mango_error.erl b/src/mango_error.erl
index 973dd43..778df2d 100644
--- a/src/mango_error.erl
+++ b/src/mango_error.erl
@@ -24,11 +24,11 @@ info(mango_cursor, {no_usable_index, operator_unsupported}) ->
         <<"no_usable_index">>,
         <<"There is no operator in this selector can used with an index.">>
     };
-info(mango_cursor, {no_usable_index, query_unsupported}) ->
+info(mango_cursor, {no_usable_index, selector_unsupported}) ->
     {
         400,
         <<"no_usable_index">>,
-        <<"Query unsupported because it would require multiple indices.">>
+        <<"There is no index available for this selector.">>
     };
 info(mango_cursor, {no_usable_index, sort_field}) ->
     {

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06bf7757/src/mango_idx.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx.erl b/src/mango_idx.erl
index c0a07a8..902fb75 100644
--- a/src/mango_idx.erl
+++ b/src/mango_idx.erl
@@ -20,6 +20,7 @@
 -export([
     list/1,
     recover/1,
+    for_sort/2,
 
     new/2,
     validate/1,
@@ -35,6 +36,7 @@
     def/1,
     opts/1,
     columns/1,
+    is_usable/2,
     start_key/2,
     end_key/2,
     cursor_mod/1,
@@ -67,6 +69,35 @@ recover(Db) ->
     end, DDocs)}.
 
 
+for_sort(Indexes, Opts) ->
+    % If a sort was specified we have to find an index that
+    % can satisfy the request.
+    case lists:keyfind(sort, 1, Opts) of
+        {sort, {SProps}} when is_list(SProps) ->
+            for_sort_int(Indexes, {SProps});
+        _ ->
+            Indexes
+    end.
+
+
+for_sort_int(Indexes, Sort) ->
+    Fields = mango_sort:fields(Sort),
+    FilterFun = fun(Idx) ->
+        Cols = mango_idx:columns(Idx),
+        case {mango_idx:type(Idx), Cols} of
+            {_, all_fields} ->
+                true;
+            {<<"text">>, _} ->
+                sets:is_subset(sets:from_list(Fields), sets:from_list(Cols));
+            {<<"json">>, _} ->
+                lists:prefix(Fields, Cols);
+            {<<"special">>, _} ->
+                lists:prefix(Fields, Cols)
+        end
+    end,
+    lists:filter(FilterFun, Indexes).
+
+
 new(Db, Opts) ->
     Def = get_idx_def(Opts),
     Type = get_idx_type(Opts),
@@ -169,6 +200,11 @@ columns(#idx{}=Idx) ->
     Mod:columns(Idx).
 
 
+is_usable(#idx{}=Idx, Selector) ->
+    Mod = idx_mod(Idx),
+    Mod:is_usable(Idx, Selector).
+
+
 start_key(#idx{}=Idx, Ranges) ->
     Mod = idx_mod(Idx),
     Mod:start_key(Ranges).

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06bf7757/src/mango_idx_special.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx_special.erl b/src/mango_idx_special.erl
index 0234dff..a8f9400 100644
--- a/src/mango_idx_special.erl
+++ b/src/mango_idx_special.erl
@@ -20,6 +20,7 @@
     from_ddoc/1,
     to_json/1,
     columns/1,
+    is_usable/2,
     start_key/1,
     end_key/1
 ]).
@@ -62,6 +63,11 @@ columns(#idx{def=all_docs}) ->
     [<<"_id">>].
 
 
+is_usable(#idx{def=all_docs}, Selector) ->
+    Fields = mango_idx_view:indexable_fields(Selector),
+    lists:member(<<"_id">>, Fields).
+
+
 start_key([{'$gt', Key, _, _}]) ->
     case mango_json:special(Key) of
         true ->

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06bf7757/src/mango_idx_view.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx_view.erl b/src/mango_idx_view.erl
index eaa4341..ce0b206 100644
--- a/src/mango_idx_view.erl
+++ b/src/mango_idx_view.erl
@@ -19,6 +19,7 @@
     remove/2,
     from_ddoc/1,
     to_json/1,
+    is_usable/2,
     columns/1,
     start_key/1,
     end_key/1,
@@ -106,6 +107,14 @@ columns(Idx) ->
     [Key || {Key, _} <- Fields].
 
 
+is_usable(Idx, Selector) ->
+    % This index is usable if at least the first column is
+    % a member of the indexable fields of the selector.
+    Columns = columns(Idx),
+    Fields = indexable_fields(Selector),
+    lists:member(hd(Columns), Fields).
+
+
 start_key([]) ->
     [];
 start_key([{'$gt', Key, _, _} | Rest]) ->


Mime
View raw message