couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tonysu...@apache.org
Subject [13/17] couchdb-mango git commit: Replace element position with brackets
Date Wed, 26 Aug 2015 16:39:14 GMT
Replace element position with brackets

To be consistent with view based indexes, we allow the user
to access array elements via position. We convert that position
to [] for the underlying indexed field. For example, myarray.0.1,
would be converted to myarray.[].[]. This converted field will then
be used for the text search. The results will then be filtered by
the match function.

Bugzid: 46268


Project: http://git-wip-us.apache.org/repos/asf/couchdb-mango/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mango/commit/460fcd47
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mango/tree/460fcd47
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mango/diff/460fcd47

Branch: refs/heads/2787-merge-repos
Commit: 460fcd4723a70ba24aee2b0254ba02602260431a
Parents: f481b56
Author: Tony Sun <tony.sun@cloudant.com>
Authored: Wed Aug 26 01:30:27 2015 -0700
Committer: Tony Sun <tony.sun@cloudant.com>
Committed: Wed Aug 26 01:30:27 2015 -0700

----------------------------------------------------------------------
 src/mango_doc.erl           | 47 +++-------------------------------
 src/mango_fields.erl        |  2 +-
 src/mango_selector_text.erl | 54 +++++++++++++++++++++++++++++++++++-----
 src/mango_util.erl          | 47 ++++++++++++++++++++++++++++++++--
 test/06-basic-text-test.py  | 22 ++++++++++++++++
 test/user_docs.py           |  4 ++-
 6 files changed, 122 insertions(+), 54 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/460fcd47/src/mango_doc.erl
----------------------------------------------------------------------
diff --git a/src/mango_doc.erl b/src/mango_doc.erl
index 479a8ad..c22b155 100644
--- a/src/mango_doc.erl
+++ b/src/mango_doc.erl
@@ -22,7 +22,6 @@
 
     get_field/2,
     get_field/3,
-    parse_field/1,
     rem_field/2,
     set_field/3
 ]).
@@ -373,7 +372,7 @@ get_field(Props, Field) ->
 
 
 get_field(Props, Field, Validator) when is_binary(Field) ->
-    {ok, Path} = parse_field(Field),
+    {ok, Path} = mango_util:parse_field(Field),
     get_field(Props, Path, Validator);
 get_field(Props, [], no_validation) ->
     Props;
@@ -411,7 +410,7 @@ get_field(_, [_|_], _) ->
 
 
 rem_field(Props, Field) when is_binary(Field) ->
-    {ok, Path} = parse_field(Field),
+    {ok, Path} = mango_util:parse_field(Field),
     rem_field(Props, Path);
 rem_field({Props}, [Name]) ->
     case lists:keytake(Name, 1, Props) of
@@ -472,7 +471,7 @@ rem_field(_, [_|_]) ->
 
 
 set_field(Props, Field, Value) when is_binary(Field) ->
-    {ok, Path} = parse_field(Field),
+    {ok, Path} = mango_util:parse_field(Field),
     set_field(Props, Path, Value);
 set_field({Props}, [Name], Value) ->
     {lists:keystore(Name, 1, Props, {Name, Value})};
@@ -536,43 +535,3 @@ set_elem(1, [_ | Rest], Value) ->
     [Value | Rest];
 set_elem(I, [Item | Rest], Value) when I > 1 ->
     [Item | set_elem(I-1, Rest, Value)].
-
-parse_field(Field) ->
-    case binary:match(Field, <<"\\">>, []) of
-        nomatch ->
-            % Fast path, no regex required
-            {ok, check_non_empty(Field, binary:split(Field, <<".">>, [global]))};
-        _ ->
-            parse_field_slow(Field)
-    end.
-
-parse_field_slow(Field) ->
-    Path = lists:map(fun
-        (P) when P =:= <<>> ->
-            ?MANGO_ERROR({invalid_field_name, Field});
-        (P) ->
-            re:replace(P, <<"\\\\">>, <<>>, [global, {return, binary}])
-    end, re:split(Field, <<"(?<!\\\\)\\.">>)),
-    {ok, Path}.
-
-check_non_empty(Field, Parts) ->
-    case lists:member(<<>>, Parts) of
-        true ->
-            ?MANGO_ERROR({invalid_field_name, Field});
-        false ->
-            Parts
-    end.
-
--ifdef(TEST).
--include_lib("eunit/include/eunit.hrl").
-
-parse_field_test() ->
-    ?assertEqual({ok, [<<"ab">>]}, parse_field(<<"ab">>)),
-    ?assertEqual({ok, [<<"a">>, <<"b">>]}, parse_field(<<"a.b">>)),
-    ?assertEqual({ok, [<<"a.b">>]}, parse_field(<<"a\\.b">>)),
-    ?assertEqual({ok, [<<"a">>, <<"b">>, <<"c">>]}, parse_field(<<"a.b.c">>)),
-    ?assertEqual({ok, [<<"a">>, <<"b.c">>]}, parse_field(<<"a.b\\.c">>)),
-    Exception = {mango_error, ?MODULE, {invalid_field_name, <<"a..b">>}},
-    ?assertThrow(Exception, parse_field(<<"a..b">>)).
-
--endif.
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/460fcd47/src/mango_fields.erl
----------------------------------------------------------------------
diff --git a/src/mango_fields.erl b/src/mango_fields.erl
index 8b6a00b..2732560 100644
--- a/src/mango_fields.erl
+++ b/src/mango_fields.erl
@@ -35,7 +35,7 @@ extract(Doc, all_fields) ->
     Doc;
 extract(Doc, Fields) ->
     lists:foldl(fun(F, NewDoc) ->
-        {ok, Path} = mango_doc:parse_field(F),
+        {ok, Path} = mango_util:parse_field(F),
         case mango_doc:get_field(Doc, Path) of
             not_found ->
                 NewDoc;

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/460fcd47/src/mango_selector_text.erl
----------------------------------------------------------------------
diff --git a/src/mango_selector_text.erl b/src/mango_selector_text.erl
index b7d78e4..e61f759 100644
--- a/src/mango_selector_text.erl
+++ b/src/mango_selector_text.erl
@@ -25,6 +25,12 @@
 -include("mango.hrl").
 
 
+%% Regex for <<"\\.">>
+-define(PERIOD, {re_pattern,0,0,<<69,82,67,80,57,0,0,0,0,0,0,0,2,0,0,0,
+0,0,0,0,46,0,0,0,48,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,93,0,
+5,27,46,84,0,5,0>>}).
+
+
 convert(Object) ->
     TupleTree = convert([], Object),
     iolist_to_binary(to_query(TupleTree)).
@@ -159,12 +165,35 @@ convert(Path, {[{<<"$size">>, Arg}]}) ->
 convert(_Path, {[{<<"$", _/binary>>=Op, _}]}) ->
     ?MANGO_ERROR({invalid_operator, Op});
 
-% We've hit a field name specifier. We need to break the name
-% into path parts and continue our conversion.
-convert(Path, {[{Field, Cond}]}) ->
-    NewPathParts = re:split(Field, <<"\\.">>),
-    NewPath = lists:reverse(NewPathParts) ++ Path,
-    convert(NewPath, Cond);
+% We've hit a field name specifier. Check if the field name is accessing
+% arrays. Convert occurrences of element position references to .[]. Then we
+% need to break the name into path parts and continue our conversion.
+convert(Path, {[{Field0, Cond}]}) ->
+    {ok, PP0} = case Field0 of
+        <<>> ->
+            {ok, []};
+        _ ->
+            mango_util:parse_field(Field0)
+    end,
+    % Later on, we perform a lucene_escape_user call on the
+    % final Path, which calls parse_field again. Calling the function
+    % twice converts <<"a\\.b">> to [<<"a">>,<<"b">>].
This leads to
+    % an incorrect query since we need [<<"a.b">>]. Without breaking
+    % our escaping mechanism, we simply revert this first parse_field
+    % effect and replace instances of "." to "\\.".
+    PP1 = [re:replace(P, ?PERIOD, <<"\\\\.">>,
+        [global,{return,binary}]) || P <- PP0],
+    {PP2, HasInteger} = replace_array_indexes(PP1, [], false),
+    NewPath = PP2 ++ Path,
+    case HasInteger of
+        true ->
+            OldPath = lists:reverse(PP1, Path),
+            OldParts = convert(OldPath, Cond),
+            NewParts = convert(NewPath, Cond),
+            {op_or, [OldParts, NewParts]};
+        false ->
+            convert(NewPath, Cond)
+    end;
 
 %% For $in
 convert(Path, Val) when is_binary(Val); is_number(Val); is_boolean(Val) ->
@@ -362,3 +391,16 @@ get_sort_types(Field, {[{_, Cond}]}, Acc)  when is_tuple(Cond)->
 
 get_sort_types(_Field, _, Acc)  ->
     Acc.
+
+
+replace_array_indexes([], NewPartsAcc, HasIntAcc) ->
+    {NewPartsAcc, HasIntAcc};
+replace_array_indexes([Part | Rest], NewPartsAcc, HasIntAcc) ->
+    {NewPart, HasInt} = try
+        _ = list_to_integer(binary_to_list(Part)),
+        {<<"[]">>, true}
+    catch _:_ ->
+        {Part, false}
+    end,
+    replace_array_indexes(Rest, [NewPart | NewPartsAcc],
+         HasInt or HasIntAcc).

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/460fcd47/src/mango_util.erl
----------------------------------------------------------------------
diff --git a/src/mango_util.erl b/src/mango_util.erl
index 0e06f66..aa11310 100644
--- a/src/mango_util.erl
+++ b/src/mango_util.erl
@@ -38,7 +38,9 @@
 
     has_suffix/2,
 
-    join/2
+    join/2,
+
+    parse_field/1
 ]).
 
 
@@ -343,7 +345,7 @@ lucene_escape_qv(<<C, Rest/binary>>) ->
 
 
 lucene_escape_user(Field) ->
-    {ok, Path} = mango_doc:parse_field(Field),
+    {ok, Path} = parse_field(Field),
     Escaped = [mango_util:lucene_escape_field(P) || P <- Path],
     iolist_to_binary(join(".", Escaped)).
 
@@ -377,3 +379,44 @@ is_number_string(Value) when is_list(Value)->
         _ ->
             true
     end.
+
+
+parse_field(Field) ->
+    case binary:match(Field, <<"\\">>, []) of
+        nomatch ->
+            % Fast path, no regex required
+            {ok, check_non_empty(Field, binary:split(Field, <<".">>, [global]))};
+        _ ->
+            parse_field_slow(Field)
+    end.
+
+parse_field_slow(Field) ->
+    Path = lists:map(fun
+        (P) when P =:= <<>> ->
+            ?MANGO_ERROR({invalid_field_name, Field});
+        (P) ->
+            re:replace(P, <<"\\\\">>, <<>>, [global, {return, binary}])
+    end, re:split(Field, <<"(?<!\\\\)\\.">>)),
+    {ok, Path}.
+
+check_non_empty(Field, Parts) ->
+    case lists:member(<<>>, Parts) of
+        true ->
+            ?MANGO_ERROR({invalid_field_name, Field});
+        false ->
+            Parts
+    end.
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+parse_field_test() ->
+    ?assertEqual({ok, [<<"ab">>]}, parse_field(<<"ab">>)),
+    ?assertEqual({ok, [<<"a">>, <<"b">>]}, parse_field(<<"a.b">>)),
+    ?assertEqual({ok, [<<"a.b">>]}, parse_field(<<"a\\.b">>)),
+    ?assertEqual({ok, [<<"a">>, <<"b">>, <<"c">>]}, parse_field(<<"a.b.c">>)),
+    ?assertEqual({ok, [<<"a">>, <<"b.c">>]}, parse_field(<<"a.b\\.c">>)),
+    Exception = {mango_error, ?MODULE, {invalid_field_name, <<"a..b">>}},
+    ?assertThrow(Exception, parse_field(<<"a..b">>)).
+
+-endif.

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/460fcd47/test/06-basic-text-test.py
----------------------------------------------------------------------
diff --git a/test/06-basic-text-test.py b/test/06-basic-text-test.py
index 28538ad..d44e340 100644
--- a/test/06-basic-text-test.py
+++ b/test/06-basic-text-test.py
@@ -59,6 +59,28 @@ class BasicTextTests(mango.UserDocsTextTests):
         assert docs[0]["name"]["first"] == "Stephanie"
         assert docs[0]["favorites"] == faves
 
+    def test_array_ref(self):
+        docs = self.db.find({"favorites.1": "Python"})
+        assert len(docs) == 4
+        for d in docs:
+            assert "Python" in d["favorites"]
+
+        # Nested Level
+        docs = self.db.find({"favorites.0.2": "Python"})
+        print len(docs)
+        assert len(docs) == 1
+        for d in docs:
+            assert "Python" in d["favorites"][0][2]
+
+    def test_number_ref(self):
+        docs = self.db.find({"11111": "number_field"})
+        assert len(docs) == 1
+        assert docs[0]["11111"] == "number_field"
+
+        docs = self.db.find({"22222.33333": "nested_number_field"})
+        assert len(docs) == 1
+        assert docs[0]["22222"]["33333"] == "nested_number_field"
+
     def test_lt(self):
         docs = self.db.find({"age": {"$lt": 22}})
         assert len(docs) == 0

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/460fcd47/test/user_docs.py
----------------------------------------------------------------------
diff --git a/test/user_docs.py b/test/user_docs.py
index baf83f7..e2f1705 100644
--- a/test/user_docs.py
+++ b/test/user_docs.py
@@ -197,7 +197,9 @@ DOCS = [
             "Erlang",
             "C",
             "Erlang"
-        ]
+        ],
+        "11111": "number_field",
+        "22222": {"33333" : "nested_number_field"}
     },
     {
         "_id": "8e1c90c0-ac18-4832-8081-40d14325bde0",


Mime
View raw message