couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From robertkowal...@apache.org
Subject couchdb-mango git commit: add mango catch-all feature
Date Wed, 10 Feb 2016 15:14:27 GMT
Repository: couchdb-mango
Updated Branches:
  refs/heads/master db3116c48 -> 01252f971


add mango catch-all feature

fall back on the all_docs index if a index for the currently
searched field does not exist.

warn about slower execution and advise to create an index.

PR: #27
PR-URL: https://github.com/apache/couchdb-mango/pull/27
Reviewed-By: Alexander Shorin
Reviewed-By: Tony Sun <tony.sun@cloudant.com>


Project: http://git-wip-us.apache.org/repos/asf/couchdb-mango/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mango/commit/01252f97
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mango/tree/01252f97
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mango/diff/01252f97

Branch: refs/heads/master
Commit: 01252f971bef0c8da1d78bf5a7b506b71926ce1b
Parents: db3116c
Author: Robert Kowalski <robertkowalski@apache.org>
Authored: Mon Jan 4 13:05:25 2016 +0100
Committer: Robert Kowalski <robertkowalski@apache.org>
Committed: Wed Feb 10 15:13:09 2016 +0000

----------------------------------------------------------------------
 src/mango_cursor.erl            | 42 ++++++++++---------------
 src/mango_cursor_special.erl    | 61 ++++++++++++++++++++++++++++++++++++
 src/mango_cursor_view.erl       |  4 ++-
 src/mango_httpd.erl             | 16 ++++++++--
 src/mango_idx.erl               | 26 +++++++++++++--
 test/02-basic-find-test.py      |  9 ++----
 test/05-index-selection-test.py | 10 ++++++
 7 files changed, 130 insertions(+), 38 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/01252f97/src/mango_cursor.erl
----------------------------------------------------------------------
diff --git a/src/mango_cursor.erl b/src/mango_cursor.erl
index 6cb7fee..911ecdb 100644
--- a/src/mango_cursor.erl
+++ b/src/mango_cursor.erl
@@ -16,7 +16,8 @@
 -export([
     create/3,
     explain/1,
-    execute/3
+    execute/3,
+    maybe_filter_indexes/2
 ]).
 
 
@@ -30,29 +31,18 @@
 
 create(Db, Selector0, Opts) ->
     Selector = mango_selector:normalize(Selector0),
-
-    ExistingIndexes = mango_idx:list(Db),
-    if ExistingIndexes /= [] -> ok; true ->
-        ?MANGO_ERROR({no_usable_index, no_indexes_defined})
-    end,
-
-    FilteredIndexes = maybe_filter_indexes(ExistingIndexes, Opts),
-    if FilteredIndexes /= [] -> ok; true ->
-        ?MANGO_ERROR({no_usable_index, no_index_matching_name})
-    end,
-
-    SortIndexes = mango_idx:for_sort(FilteredIndexes, Opts),
-    if SortIndexes /= [] -> ok; true ->
-        ?MANGO_ERROR({no_usable_index, missing_sort_index})
-    end,
-
-    UsableFilter = fun(I) -> mango_idx:is_usable(I, Selector) end,
-    UsableIndexes = lists:filter(UsableFilter, SortIndexes),
-    if UsableIndexes /= [] -> ok; true ->
-        ?MANGO_ERROR({no_usable_index, selector_unsupported})
-    end,
-
-    create_cursor(Db, UsableIndexes, Selector, Opts).
+    UsableIndexes = mango_idx:get_usable_indexes(Db, Selector0, Opts),
+
+    {use_index, IndexSpecified} = proplists:lookup(use_index, Opts),
+    case {length(UsableIndexes), length(IndexSpecified)} of
+        {0, 1} ->
+            ?MANGO_ERROR({no_usable_index, selector_unsupported});
+        {0, 0} ->
+            AllDocs = mango_idx:special(Db),
+            create_cursor(Db, AllDocs, Selector, Opts);
+        _ ->
+            create_cursor(Db, UsableIndexes, Selector, Opts)
+    end.
 
 
 explain(#cursor{}=Cursor) ->
@@ -125,9 +115,9 @@ group_indexes_by_type(Indexes) ->
     % queries.
     CursorModules = case module_loaded(dreyfus_index) of
         true ->
-            [mango_cursor_view, mango_cursor_text];
+            [mango_cursor_view, mango_cursor_text, mango_cursor_special];
         false ->
-            [mango_cursor_view]
+            [mango_cursor_view, mango_cursor_special]
     end,
     lists:flatmap(fun(CMod) ->
         case dict:find(CMod, IdxDict) of

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/01252f97/src/mango_cursor_special.erl
----------------------------------------------------------------------
diff --git a/src/mango_cursor_special.erl b/src/mango_cursor_special.erl
new file mode 100644
index 0000000..6058217
--- /dev/null
+++ b/src/mango_cursor_special.erl
@@ -0,0 +1,61 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_cursor_special).
+
+-export([
+    create/4,
+    explain/1,
+    execute/3
+]).
+
+-export([
+    handle_message/2
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch_mrview/include/couch_mrview.hrl").
+-include("mango_cursor.hrl").
+
+
+create(Db, Indexes, Selector, Opts) ->
+    InitialRange = mango_idx_view:field_ranges(Selector),
+    CatchAll = [{<<"_id">>, {'$gt', null, '$lt', mango_json_max}}],
+    FieldRanges = lists:append(CatchAll, InitialRange),
+    Composited = mango_cursor_view:composite_indexes(Indexes, FieldRanges),
+    {Index, IndexRanges} = mango_cursor_view:choose_best_index(Db, Composited),
+
+    Limit = couch_util:get_value(limit, Opts, 10000000000),
+    Skip = couch_util:get_value(skip, Opts, 0),
+    Fields = couch_util:get_value(fields, Opts, all_fields),
+
+    {ok, #cursor{
+        db = Db,
+        index = Index,
+        ranges = IndexRanges,
+        selector = Selector,
+        opts = Opts,
+        limit = Limit,
+        skip = Skip,
+        fields = Fields
+    }}.
+
+
+explain(Cursor) ->
+    mango_cursor_view:explain(Cursor).
+
+execute(Cursor0, UserFun, UserAcc) ->
+    mango_cursor_view:execute(Cursor0, UserFun, UserAcc).
+
+handle_message(Msg, Cursor) ->
+    mango_cursor_view:handle_message(Msg, Cursor).

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/01252f97/src/mango_cursor_view.erl
----------------------------------------------------------------------
diff --git a/src/mango_cursor_view.erl b/src/mango_cursor_view.erl
index cb35968..5f109cd 100644
--- a/src/mango_cursor_view.erl
+++ b/src/mango_cursor_view.erl
@@ -19,7 +19,9 @@
 ]).
 
 -export([
-    handle_message/2
+    handle_message/2,
+    composite_indexes/2,
+    choose_best_index/2
 ]).
 
 

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/01252f97/src/mango_httpd.erl
----------------------------------------------------------------------
diff --git a/src/mango_httpd.erl b/src/mango_httpd.erl
index ef7135c..bde3850 100644
--- a/src/mango_httpd.erl
+++ b/src/mango_httpd.erl
@@ -181,7 +181,7 @@ handle_find_req(#httpd{method='POST'}=Req, Db) ->
     chttpd:validate_ctype(Req, "application/json"),
     {ok, Opts0} = mango_opts:validate_find(chttpd:json_body_obj(Req)),
     {value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0),
-    {ok, Resp0} = start_find_resp(Req),
+    {ok, Resp0} = start_find_resp(Req, Db, Sel, Opts),
     {ok, AccOut} = run_find(Resp0, Db, Sel, Opts),
     end_find_resp(AccOut);
 
@@ -228,8 +228,18 @@ convert_to_design_id(DDocId) ->
     end.
 
 
-start_find_resp(Req) ->
-    chttpd:start_delayed_json_response(Req, 200, [], "{\"docs\":[").
+start_find_resp(Req, Db, Sel, Opts) ->
+    chttpd:start_delayed_json_response(Req, 200, [], maybe_add_warning(Db, Sel, Opts)).
+
+
+maybe_add_warning(Db, Selector, Opts) ->
+    UsableIndexes = mango_idx:get_usable_indexes(Db, Selector, Opts),
+    case length(UsableIndexes) of
+        0 ->
+            "{\"warning\":\"no matching index found, create an index to optimize query time\",\r\n\"docs\":[";
+        _ ->
+            "{\"docs\":["
+    end.
 
 
 end_find_resp(Acc0) ->

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/01252f97/src/mango_idx.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx.erl b/src/mango_idx.erl
index 377af91..11c713b 100644
--- a/src/mango_idx.erl
+++ b/src/mango_idx.erl
@@ -42,7 +42,8 @@
     cursor_mod/1,
     idx_mod/1,
     to_json/1,
-    delete/4
+    delete/4,
+    get_usable_indexes/3
 ]).
 
 
@@ -55,6 +56,27 @@ list(Db) ->
     {ok, Indexes} = ddoc_cache:open(db_to_name(Db), ?MODULE),
     Indexes.
 
+get_usable_indexes(Db, Selector0, Opts) ->
+    Selector = mango_selector:normalize(Selector0),
+
+    ExistingIndexes = mango_idx:list(Db),
+    if ExistingIndexes /= [] -> ok; true ->
+        ?MANGO_ERROR({no_usable_index, no_indexes_defined})
+    end,
+
+    FilteredIndexes = mango_cursor:maybe_filter_indexes(ExistingIndexes, Opts),
+    if FilteredIndexes /= [] -> ok; true ->
+        ?MANGO_ERROR({no_usable_index, no_index_matching_name})
+    end,
+
+    SortIndexes = mango_idx:for_sort(FilteredIndexes, Opts),
+    if SortIndexes /= [] -> ok; true ->
+        ?MANGO_ERROR({no_usable_index, missing_sort_index})
+    end,
+
+    UsableFilter = fun(I) -> mango_idx:is_usable(I, Selector) end,
+    lists:filter(UsableFilter, SortIndexes).
+
 recover(Db) ->
     {ok, DDocs0} = mango_util:open_ddocs(Db),
     Pred = fun({Props}) ->
@@ -245,7 +267,7 @@ end_key(#idx{}=Idx, Ranges) ->
 cursor_mod(#idx{type = <<"json">>}) ->
     mango_cursor_view;
 cursor_mod(#idx{def = all_docs, type= <<"special">>}) ->
-    mango_cursor_view;
+    mango_cursor_special;
 cursor_mod(#idx{type = <<"text">>}) ->
     case module_loaded(dreyfus_index) of
         true ->

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/01252f97/test/02-basic-find-test.py
----------------------------------------------------------------------
diff --git a/test/02-basic-find-test.py b/test/02-basic-find-test.py
index 37459a2..c7eb348 100644
--- a/test/02-basic-find-test.py
+++ b/test/02-basic-find-test.py
@@ -236,12 +236,9 @@ class BasicFindTests(mango.UserDocsTests):
             assert len(docs) == 15
 
     def test_empty(self):
-        try:
-            self.db.find({})
-        except Exception, e:
-            assert e.response.status_code == 400
-        else:
-            raise AssertionError("bad find")
+        docs = self.db.find({})
+        # 15 users and 9 design docs
+        assert len(docs) == 24
 
     def test_empty_subsel(self):
         docs = self.db.find({

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/01252f97/test/05-index-selection-test.py
----------------------------------------------------------------------
diff --git a/test/05-index-selection-test.py b/test/05-index-selection-test.py
index 3a757f8..bbd3aa7 100644
--- a/test/05-index-selection-test.py
+++ b/test/05-index-selection-test.py
@@ -74,6 +74,16 @@ class IndexSelectionTests(mango.UserDocsTests):
             }, use_index=ddocid, explain=True)
         assert resp["index"]["ddoc"] == ddocid
 
+    def test_use_most_columns(self):
+        # ddoc id for the age index
+        ddocid = "_design/ad3d537c03cd7c6a43cf8dff66ef70ea54c2b40f"
+        try:
+            self.db.find({}, use_index=ddocid)
+        except Exception, e:
+            assert e.response.status_code == 400
+        else:
+            raise AssertionError("bad find")
+
     # This doc will not be saved given the new ddoc validation code
     # in couch_mrview
     def test_manual_bad_view_idx01(self):


Mime
View raw message