couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From willhol...@apache.org
Subject [couchdb] branch master updated: Fix maximum key value when using JSON indexes (#881)
Date Mon, 16 Oct 2017 15:01:50 GMT
This is an automated email from the ASF dual-hosted git repository.

willholley pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/couchdb.git


The following commit(s) were added to refs/heads/master by this push:
     new 641aa56  Fix maximum key value when using JSON indexes (#881)
641aa56 is described below

commit 641aa568d011100aacdee6128da51b5a98fdbd8f
Author: Will Holley <willholley@gmail.com>
AuthorDate: Mon Oct 16 16:01:47 2017 +0100

    Fix maximum key value when using JSON indexes (#881)
    
    Mango previously constrained range queries against JSON indexes
    (map/reduce views) to startkey=[]&endkey=[{}]. In Mango, JSON
    index keys are always compound (i.e. always arrays), but this
    restriction resulted in Mango failing to match documents where
    the indexed value was an object.
    
    For example, an index with keys:
    
    [1],
    [2],
    [{"foo": 3}]
    
    would be restricted such that only [1] and [2] were returned
    if a range query was issued.
    
    On its own, this behaviour isn't necessarily unintuitive, but
    it is different from the behaviour of a non-indexed Mango
    query, so the query results would change in the presence of an
    index.
    
    Additonally, it prevented operators or selectors which explicitly
    depend on a full index scan (such as $exists) from returning a
    complete result set.
    
    This commit changes the maximum range boundary from {} to a
    value that collates higher than any JSON object, so all
    array/compound keys will be included.
    
    Note that this uses an invalid UTF-8 character, so we depend
    on the view engine not barfing when this is passed as a
    parameter. In addition, we can't represent the value in JSON
    so we need to subtitute is when returning a query plan
    in the _explain endpoint.
---
 src/mango/src/mango_cursor_view.erl        | 23 +++++++-
 src/mango/src/mango_idx_view.erl           |  5 +-
 src/mango/src/mango_idx_view.hrl           | 13 +++++
 src/mango/test/02-basic-find-test.py       |  2 +-
 src/mango/test/17-multi-type-value-test.py | 90 ++++++++++++++++++++++++++++++
 5 files changed, 128 insertions(+), 5 deletions(-)

diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 59dd522..3fcec07 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -29,7 +29,7 @@
 -include_lib("couch/include/couch_db.hrl").
 -include_lib("couch_mrview/include/couch_mrview.hrl").
 -include("mango_cursor.hrl").
-
+-include("mango_idx_view.hrl").
 
 create(Db, Indexes, Selector, Opts) ->
     FieldRanges = mango_idx_view:field_ranges(Selector),
@@ -61,18 +61,37 @@ explain(Cursor) ->
 
     BaseArgs = base_args(Cursor),
     Args = apply_opts(Opts, BaseArgs),
+
     [{mrargs, {[
         {include_docs, Args#mrargs.include_docs},
         {view_type, Args#mrargs.view_type},
         {reduce, Args#mrargs.reduce},
         {start_key, Args#mrargs.start_key},
-        {end_key, Args#mrargs.end_key},
+        {end_key, maybe_replace_max_json(Args#mrargs.end_key)},
         {direction, Args#mrargs.direction},
         {stable, Args#mrargs.stable},
         {update, Args#mrargs.update}
     ]}}].
 
 
+% replace internal values that cannot
+% be represented as a valid UTF-8 string
+% with a token for JSON serialization
+maybe_replace_max_json([]) ->
+    [];
+
+maybe_replace_max_json(?MAX_STR) ->
+    <<"<MAX>">>;
+
+maybe_replace_max_json([H | T] = EndKey) when is_list(EndKey) ->
+    H1 = if H == ?MAX_JSON_OBJ -> <<"<MAX>">>;
+            true -> H
+    end,
+    [H1 | maybe_replace_max_json(T)];
+
+maybe_replace_max_json(EndKey) ->
+    EndKey.
+
 base_args(#cursor{index = Idx} = Cursor) ->
     #mrargs{
         view_type = map,
diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl
index 8331683..f1041bb 100644
--- a/src/mango/src/mango_idx_view.erl
+++ b/src/mango/src/mango_idx_view.erl
@@ -34,6 +34,7 @@
 -include_lib("couch/include/couch_db.hrl").
 -include("mango.hrl").
 -include("mango_idx.hrl").
+-include("mango_idx_view.hrl").
 
 
 validate_new(#idx{}=Idx, _Db) ->
@@ -163,11 +164,11 @@ start_key([{'$eq', Key, '$eq', Key} | Rest]) ->
 
 
 end_key([]) ->
-    [{[]}];
+    [?MAX_JSON_OBJ];
 end_key([{_, _, '$lt', Key} | Rest]) ->
     case mango_json:special(Key) of
         true ->
-            [{[]}];
+            [?MAX_JSON_OBJ];
         false ->
             [Key | end_key(Rest)]
     end;
diff --git a/src/mango/src/mango_idx_view.hrl b/src/mango/src/mango_idx_view.hrl
new file mode 100644
index 0000000..0d213e5
--- /dev/null
+++ b/src/mango/src/mango_idx_view.hrl
@@ -0,0 +1,13 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-define(MAX_JSON_OBJ, {<<255, 255, 255, 255>>}).
\ No newline at end of file
diff --git a/src/mango/test/02-basic-find-test.py b/src/mango/test/02-basic-find-test.py
index 72a4e3f..82554a1 100644
--- a/src/mango/test/02-basic-find-test.py
+++ b/src/mango/test/02-basic-find-test.py
@@ -319,5 +319,5 @@ class BasicFindTests(mango.UserDocsTests):
         assert explain["mrargs"]["update"] == True
         assert explain["mrargs"]["reduce"] == False
         assert explain["mrargs"]["start_key"] == [0]
-        assert explain["mrargs"]["end_key"] == [{}]
+        assert explain["mrargs"]["end_key"] == ["<MAX>"]
         assert explain["mrargs"]["include_docs"] == True
diff --git a/src/mango/test/17-multi-type-value-test.py b/src/mango/test/17-multi-type-value-test.py
new file mode 100644
index 0000000..d838447
--- /dev/null
+++ b/src/mango/test/17-multi-type-value-test.py
@@ -0,0 +1,90 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import copy
+import mango
+import unittest
+
+DOCS = [
+    {
+        "_id": "1",
+        "name": "Jimi",
+        "age": 10
+    },
+    {
+        "_id": "2",
+        "name": {"forename":"Eddie"},
+        "age": 20
+    },
+    {
+        "_id": "3",
+        "name": None,
+        "age": 30
+    },
+    {
+        "_id": "4",
+        "name": 1,
+        "age": 40
+    },
+    {
+        "_id": "5",
+        "forename": "Sam",
+        "age": 50
+    }
+]
+
+
+class MultiValueFieldTests:
+
+    def test_can_query_with_name(self):
+        docs = self.db.find({"name": {"$exists": True}})
+        self.assertEqual(len(docs), 4)
+        for d in docs:
+            self.assertIn("name", d)
+
+    def test_can_query_with_name_subfield(self):
+        docs = self.db.find({"name.forename": {"$exists": True}})
+        self.assertEqual(len(docs), 1)
+        self.assertEqual(docs[0]["_id"], "2")
+
+    def test_can_query_with_name_range(self):
+        docs = self.db.find({"name": {"$gte": 0}})
+        # expect to include "Jimi", 1 and {"forename":"Eddie"}
+        self.assertEqual(len(docs), 3)
+        for d in docs:
+            self.assertIn("name", d)
+
+    def test_can_query_with_age_and_name_range(self):
+        docs = self.db.find({"age": {"$gte": 0, "$lt": 40}, "name": {"$gte": 0}})
+        # expect to include "Jimi", 1 and {"forename":"Eddie"}
+        self.assertEqual(len(docs), 2)
+        for d in docs:
+            self.assertIn("name", d)
+
+
+
+class MultiValueFieldJSONTests(mango.DbPerClass, MultiValueFieldTests):
+    def setUp(self):
+        self.db.recreate()
+        self.db.save_docs(copy.deepcopy(DOCS))
+        self.db.create_index(["name"])
+        self.db.create_index(["age", "name"])
+
+# @unittest.skipUnless(mango.has_text_service(), "requires text service")
+# class MultiValueFieldTextTests(MultiValueFieldDocsNoIndexes, OperatorTests):
+#     pass
+
+
+class MultiValueFieldAllDocsTests(mango.DbPerClass, MultiValueFieldTests):
+    def setUp(self):
+        self.db.recreate()
+        self.db.save_docs(copy.deepcopy(DOCS))

-- 
To stop receiving notification emails like this one, please contact
['"commits@couchdb.apache.org" <commits@couchdb.apache.org>'].

Mime
View raw message