couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tonysu...@apache.org
Subject couchdb-mango git commit: Handle malformed manual design docs.
Date Wed, 16 Sep 2015 23:58:38 GMT
Repository: couchdb-mango
Updated Branches:
  refs/heads/2816-malformed-docs [created] 2d26f2295


Handle malformed manual design docs.

We separate index validation into three phases. The first phase is
index creation via our _index api. This validation piece will throw
an error for invalid index definitions. The second phase is during
the indexing of documents. If an index definition is not valid, we
will not use the definition to index documents. We silently log the
error. Finally, during the query phase, design documents are again
validated to ensure correct indexes are used. Again, we log an error
but silently ignore invalid index definitions. Our validation will
integrate into a consolidated validation of all indexers.

COUCHDB-2816


Project: http://git-wip-us.apache.org/repos/asf/couchdb-mango/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mango/commit/2d26f229
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mango/tree/2d26f229
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mango/diff/2d26f229

Branch: refs/heads/2816-malformed-docs
Commit: 2d26f22954eb99a92fec32b930169eb7c538753f
Parents: f924032
Author: Tony Sun <tony.sun@cloudant.com>
Authored: Wed Sep 16 16:42:47 2015 -0700
Committer: Tony Sun <tony.sun@cloudant.com>
Committed: Wed Sep 16 16:42:47 2015 -0700

----------------------------------------------------------------------
 src/mango_httpd.erl             |  2 +-
 src/mango_idx.erl               |  6 +--
 src/mango_idx_text.erl          | 42 ++++++++++++++++-----
 src/mango_idx_view.erl          | 46 ++++++++++++++++-------
 src/mango_native_proc.erl       | 38 ++++++++++++++++++-
 test/05-index-selection-test.py | 71 ++++++++++++++++++++++++++++++++++++
 6 files changed, 176 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/2d26f229/src/mango_httpd.erl
----------------------------------------------------------------------
diff --git a/src/mango_httpd.erl b/src/mango_httpd.erl
index 8ac3d3c..ef83bda 100644
--- a/src/mango_httpd.erl
+++ b/src/mango_httpd.erl
@@ -83,7 +83,7 @@ handle_index_req(#httpd{method='GET', path_parts=[_, _]}=Req, Db) ->
 handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) ->
     {ok, Opts} = mango_opts:validate_idx_create(chttpd:json_body_obj(Req)),
     {ok, Idx0} = mango_idx:new(Db, Opts),
-    {ok, Idx} = mango_idx:validate(Idx0),
+    {ok, Idx} = mango_idx:validate_new(Idx0),
     {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx)),
     Id = Idx#idx.ddoc,
     Name = Idx#idx.name,

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/2d26f229/src/mango_idx.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx.erl b/src/mango_idx.erl
index 56b7fc2..4118d22 100644
--- a/src/mango_idx.erl
+++ b/src/mango_idx.erl
@@ -23,7 +23,7 @@
     for_sort/2,
 
     new/2,
-    validate/1,
+    validate_new/1,
     add/2,
     remove/2,
     from_ddoc/2,
@@ -114,9 +114,9 @@ new(Db, Opts) ->
     }}.
 
 
-validate(Idx) ->
+validate_new(Idx) ->
     Mod = idx_mod(Idx),
-    Mod:validate(Idx).
+    Mod:validate_new(Idx).
 
 
 add(DDoc, Idx) ->

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/2d26f229/src/mango_idx_text.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx_text.erl b/src/mango_idx_text.erl
index 321889a..fcd2939 100644
--- a/src/mango_idx_text.erl
+++ b/src/mango_idx_text.erl
@@ -14,8 +14,9 @@
 
 
 -export([
-    validate/1,
+    validate_new/1,
     validate_fields/1,
+    validate_index_def/1,
     add/2,
     remove/2,
     from_ddoc/1,
@@ -31,11 +32,15 @@
 -include("mango_idx.hrl").
 
 
-validate(#idx{}=Idx) ->
+validate_new(#idx{}=Idx) ->
     {ok, Def} = do_validate(Idx#idx.def),
     {ok, Idx#idx{def=Def}}.
 
 
+validate_index_def(IndexInfo) ->
+    do_validate(IndexInfo).
+
+
 add(#doc{body={Props0}}=DDoc, Idx) ->
     Texts1 = case proplists:get_value(<<"indexes">>, Props0) of
         {Texts0} -> Texts0;
@@ -72,14 +77,17 @@ from_ddoc({Props}) ->
     case lists:keyfind(<<"indexes">>, 1, Props) of
         {<<"indexes">>, {Texts}} when is_list(Texts) ->
             lists:flatmap(fun({Name, {VProps}}) ->
-                Def = proplists:get_value(<<"index">>, VProps),
-                I = #idx{
-                    type = <<"text">>,
-                    name = Name,
-                    def = Def
-                },
-                % TODO: Validate the index definition
-                [I]
+                case validate_ddoc(VProps) of
+                    invalid_ddoc ->
+                        [];
+                    Def ->
+                        I = #idx{
+                        type = <<"text">>,
+                        name = Name,
+                        def = Def
+                        },
+                        [I]
+                end
             end, Texts);
         _ ->
             []
@@ -165,6 +173,8 @@ validate_field_type(<<"boolean">>) ->
     <<"boolean">>.
 
 
+validate_fields(<<"all_fields">>) ->
+    {ok, all_fields};
 validate_fields(Fields) ->
     try fields_to_json(Fields) of
         _ ->
@@ -174,6 +184,18 @@ validate_fields(Fields) ->
     end.
 
 
+validate_ddoc(VProps) ->
+    try
+        Def = proplists:get_value(<<"index">>, VProps),
+        validate_index_def(Def),
+        Def
+    catch Error:Reason ->
+        couch_log:error("Invalid Index Def ~p: Error. ~p, Reason: ~p",
+            [VProps, Error, Reason]),
+        invalid_ddoc
+    end.
+
+
 opts() ->
     [
         {<<"default_analyzer">>, [

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/2d26f229/src/mango_idx_view.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx_view.erl b/src/mango_idx_view.erl
index cc5bcb3..0cc713f 100644
--- a/src/mango_idx_view.erl
+++ b/src/mango_idx_view.erl
@@ -14,7 +14,8 @@
 
 
 -export([
-    validate/1,
+    validate_new/1,
+    validate_index_def/1,
     add/2,
     remove/2,
     from_ddoc/1,
@@ -35,11 +36,15 @@
 -include("mango_idx.hrl").
 
 
-validate(#idx{}=Idx) ->
+validate_new(#idx{}=Idx) ->
     {ok, Def} = do_validate(Idx#idx.def),
     {ok, Idx#idx{def=Def}}.
 
 
+validate_index_def(Def) ->
+    def_to_json(Def).
+
+
 add(#doc{body={Props0}}=DDoc, Idx) ->
     Views1 = case proplists:get_value(<<"views">>, Props0) of
         {Views0} -> Views0;
@@ -75,17 +80,18 @@ from_ddoc({Props}) ->
     case lists:keyfind(<<"views">>, 1, Props) of
         {<<"views">>, {Views}} when is_list(Views) ->
             lists:flatmap(fun({Name, {VProps}}) ->
-                Def = proplists:get_value(<<"map">>, VProps),
-                {Opts0} = proplists:get_value(<<"options">>, VProps),
-                Opts = lists:keydelete(<<"sort">>, 1, Opts0),
-                I = #idx{
-                    type = <<"json">>,
-                    name = Name,
-                    def = Def,
-                    opts = Opts
-                },
-                % TODO: Validate the index definition
-                [I]
+                case validate_ddoc(VProps) of
+                    invalid_view ->
+                        [];
+                    {Def, Opts} ->
+                        I = #idx{
+                        type = <<"json">>,
+                        name = Name,
+                        def = Def,
+                        opts = Opts
+                        },
+                        [I]
+                end
             end, Views);
         _ ->
             []
@@ -204,6 +210,20 @@ make_view(Idx) ->
     {Idx#idx.name, View}.
 
 
+validate_ddoc(VProps) ->
+    try
+        Def = proplists:get_value(<<"map">>, VProps),
+        validate_index_def(Def),
+        {Opts0} = proplists:get_value(<<"options">>, VProps),
+        Opts = lists:keydelete(<<"sort">>, 1, Opts0),
+        {Def, Opts}
+    catch Error:Reason ->
+        couch_log:error("Invalid Index Def ~p. Error: ~p, Reason: ~p",
+            [VProps, Error, Reason]),
+        invalid_view
+    end.
+
+
 % This function returns a list of indexes that
 % can be used to restrict this query. This works by
 % searching the selector looking for field names that

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/2d26f229/src/mango_native_proc.erl
----------------------------------------------------------------------
diff --git a/src/mango_native_proc.erl b/src/mango_native_proc.erl
index 9481ca4..822d173 100644
--- a/src/mango_native_proc.erl
+++ b/src/mango_native_proc.erl
@@ -14,6 +14,9 @@
 -behavior(gen_server).
 
 
+-include("mango_idx.hrl").
+
+
 -export([
     start_link/0,
     set_timeout/2,
@@ -72,7 +75,13 @@ handle_call({prompt, [<<"reset">>, _QueryConfig]}, _From, St)
->
     {reply, true, St#st{indexes=[]}};
 
 handle_call({prompt, [<<"add_fun">>, IndexInfo]}, _From, St) ->
-    Indexes = St#st.indexes ++ [IndexInfo],
+    Indexes = case validate_index_info(IndexInfo) of
+        true ->
+            St#st.indexes ++ [IndexInfo];
+        false ->
+            couch_log:error("No Valid Indexes For: ~p", [IndexInfo]),
+            St#st.indexes
+    end,
     NewSt = St#st{indexes = Indexes},
     {reply, true, NewSt};
 
@@ -86,7 +95,14 @@ handle_call({prompt, [<<"rereduce">>, _, _]}, _From, St) ->
     {reply, null, St};
 
 handle_call({prompt, [<<"index_doc">>, Doc]}, _From, St) ->
-    {reply, index_doc(St, mango_json:to_binary(Doc)), St};
+    Vals = case index_doc(St, mango_json:to_binary(Doc)) of
+        [] ->
+            [[]];
+        Else ->
+            Else
+    end,
+    {reply, Vals, St};
+
 
 handle_call(Msg, _From, St) ->
     {stop, {invalid_call, Msg}, {invalid_call, Msg}, St}.
@@ -296,3 +312,21 @@ make_text_field_name([P | Rest], Type) ->
     Parts = lists:reverse(Rest, [iolist_to_binary([P, ":", Type])]),
     Escaped = [mango_util:lucene_escape_field(N) || N <- Parts],
     iolist_to_binary(mango_util:join(".", Escaped)).
+
+
+validate_index_info(IndexInfo) ->
+    IdxTypes = case module_loaded(dreyfus_index) of
+        true ->
+            [mango_idx_view, mango_idx_text];
+        false ->
+            [mango_idx_view]
+    end,
+    Results = lists:foldl(fun(IdxType, Results0) ->
+        try
+            IdxType:validate_index_def(IndexInfo),
+            [valid_index | Results0]
+        catch _:_ ->
+            [invalid_index | Results0]
+        end
+    end, [], IdxTypes),
+    lists:member(valid_index, Results).
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/2d26f229/test/05-index-selection-test.py
----------------------------------------------------------------------
diff --git a/test/05-index-selection-test.py b/test/05-index-selection-test.py
index eecaf17..3652bc4 100644
--- a/test/05-index-selection-test.py
+++ b/test/05-index-selection-test.py
@@ -74,6 +74,77 @@ class IndexSelectionTests(mango.UserDocsTests):
             }, use_index=ddocid, explain=True)
         assert resp["index"]["ddoc"] == ddocid
 
+    # Silently log error and use good index def in ddoc
+    def test_manual_bad_view_idx01(self):
+        design_doc = {
+            "_id": "_design/bad_view_index",
+            "language": "query",
+            "views": {
+                "queryidx1": {
+                    "map": {
+                        "fields": {
+                            "age": "asc"
+                        }
+                    },
+                    "reduce": "_count",
+                    "options": {
+                        "def": {
+                            "fields": [
+                                {
+                                    "age": "asc"
+                                }
+                            ]
+                        },
+                        "w": 2
+                    }
+                }
+            },
+            "views" : {
+                "views001" : {
+                "map" : "function(employee){if(employee.training)"
+                    + "{emit(employee.number, employee.training);}}"
+                }
+            }
+        }
+        self.db.save_doc(design_doc)
+        docs= self.db.find({"age" : 48})
+        assert len(docs) == 1
+        assert docs[0]["name"]["first"] == "Stephanie"
+        assert docs[0]["age"] == 48
+
+    @unittest.skipUnless(mango.has_text_service(), "requires text service")
+    def test_manual_bad_text_idx(self):
+        design_doc = {
+            "_id": "_design/bad_text_index",
+            "language": "query",
+            "indexes": {
+                    "text_index": {
+                        "default_analyzer": "keyword",
+                        "default_field": {},
+                        "selector": {},
+                        "fields": "all_fields",
+                        "analyzer": {
+                        "name": "perfield",
+                        "default": "keyword",
+                        "fields": {
+                            "$default": "standard"
+                        }
+                    }
+                }
+            },
+            "indexes": {
+                "st_index": {
+                    "analyzer": "standard",
+                    "index": "function(doc){\n index(\"st_index\", doc.geometry);\n}"
+                }
+            }
+        }
+        self.db.save_doc(design_doc)
+        docs= self.db.find({"age" : 48})
+        assert len(docs) == 1
+        assert docs[0]["name"]["first"] == "Stephanie"
+        assert docs[0]["age"] == 48
+
 
 @unittest.skipUnless(mango.has_text_service(), "requires text service")
 class MultiTextIndexSelectionTests(mango.UserDocsTests):


Mime
View raw message