Return-Path: X-Original-To: apmail-couchdb-commits-archive@www.apache.org Delivered-To: apmail-couchdb-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id ED98E18F14 for ; Mon, 5 Oct 2015 20:57:48 +0000 (UTC) Received: (qmail 97269 invoked by uid 500); 5 Oct 2015 20:57:39 -0000 Delivered-To: apmail-couchdb-commits-archive@couchdb.apache.org Received: (qmail 97223 invoked by uid 500); 5 Oct 2015 20:57:39 -0000 Mailing-List: contact commits-help@couchdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@couchdb.apache.org Delivered-To: mailing list commits@couchdb.apache.org Received: (qmail 97214 invoked by uid 99); 5 Oct 2015 20:57:39 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 05 Oct 2015 20:57:39 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 44AA1E03C8; Mon, 5 Oct 2015 20:57:39 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: tonysun83@apache.org To: commits@couchdb.apache.org Message-Id: <6825658e67664c819cd305697fcbd35d@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: couchdb-mango git commit: Handle malformed manual design docs. Date: Mon, 5 Oct 2015 20:57:39 +0000 (UTC) Repository: couchdb-mango Updated Branches: refs/heads/master ea662ba96 -> 06cf106dc Handle malformed manual design docs. We separate index validation into three phases. The first phase is index creation via our _index api. This validation piece will throw an error for invalid index definitions. The second phase is during the indexing of documents. If an index definition is not valid, we will not use the definition to index documents. We silently log the error. Finally, during the query phase, design documents are again validated to ensure correct indexes are used. Again, we log an error but silently ignore invalid index definitions. Our validation will integrate into a consolidated validation of all indexers. COUCHDB-2816 Project: http://git-wip-us.apache.org/repos/asf/couchdb-mango/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mango/commit/06cf106d Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mango/tree/06cf106d Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mango/diff/06cf106d Branch: refs/heads/master Commit: 06cf106dc47188ce399b62c82c786197ef85a382 Parents: ea662ba Author: Tony Sun Authored: Wed Sep 16 16:42:47 2015 -0700 Committer: Tony Sun Committed: Mon Oct 5 13:45:10 2015 -0700 ---------------------------------------------------------------------- src/mango_httpd.erl | 2 +- src/mango_idx.erl | 6 ++-- src/mango_idx_text.erl | 42 ++++++++++++++++------ src/mango_idx_view.erl | 46 +++++++++++++++++------- src/mango_native_proc.erl | 38 ++++++++++++++++++-- test/05-index-selection-test.py | 69 ++++++++++++++++++++++++++++++++++++ 6 files changed, 174 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06cf106d/src/mango_httpd.erl ---------------------------------------------------------------------- diff --git a/src/mango_httpd.erl b/src/mango_httpd.erl index 8ac3d3c..ef83bda 100644 --- a/src/mango_httpd.erl +++ b/src/mango_httpd.erl @@ -83,7 +83,7 @@ handle_index_req(#httpd{method='GET', path_parts=[_, _]}=Req, Db) -> handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) -> {ok, Opts} = mango_opts:validate_idx_create(chttpd:json_body_obj(Req)), {ok, Idx0} = mango_idx:new(Db, Opts), - {ok, Idx} = mango_idx:validate(Idx0), + {ok, Idx} = mango_idx:validate_new(Idx0), {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx)), Id = Idx#idx.ddoc, Name = Idx#idx.name, http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06cf106d/src/mango_idx.erl ---------------------------------------------------------------------- diff --git a/src/mango_idx.erl b/src/mango_idx.erl index 867cbef..377af91 100644 --- a/src/mango_idx.erl +++ b/src/mango_idx.erl @@ -23,7 +23,7 @@ for_sort/2, new/2, - validate/1, + validate_new/1, add/2, remove/2, from_ddoc/2, @@ -114,9 +114,9 @@ new(Db, Opts) -> }}. -validate(Idx) -> +validate_new(Idx) -> Mod = idx_mod(Idx), - Mod:validate(Idx). + Mod:validate_new(Idx). add(DDoc, Idx) -> http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06cf106d/src/mango_idx_text.erl ---------------------------------------------------------------------- diff --git a/src/mango_idx_text.erl b/src/mango_idx_text.erl index 321889a..fcd2939 100644 --- a/src/mango_idx_text.erl +++ b/src/mango_idx_text.erl @@ -14,8 +14,9 @@ -export([ - validate/1, + validate_new/1, validate_fields/1, + validate_index_def/1, add/2, remove/2, from_ddoc/1, @@ -31,11 +32,15 @@ -include("mango_idx.hrl"). -validate(#idx{}=Idx) -> +validate_new(#idx{}=Idx) -> {ok, Def} = do_validate(Idx#idx.def), {ok, Idx#idx{def=Def}}. +validate_index_def(IndexInfo) -> + do_validate(IndexInfo). + + add(#doc{body={Props0}}=DDoc, Idx) -> Texts1 = case proplists:get_value(<<"indexes">>, Props0) of {Texts0} -> Texts0; @@ -72,14 +77,17 @@ from_ddoc({Props}) -> case lists:keyfind(<<"indexes">>, 1, Props) of {<<"indexes">>, {Texts}} when is_list(Texts) -> lists:flatmap(fun({Name, {VProps}}) -> - Def = proplists:get_value(<<"index">>, VProps), - I = #idx{ - type = <<"text">>, - name = Name, - def = Def - }, - % TODO: Validate the index definition - [I] + case validate_ddoc(VProps) of + invalid_ddoc -> + []; + Def -> + I = #idx{ + type = <<"text">>, + name = Name, + def = Def + }, + [I] + end end, Texts); _ -> [] @@ -165,6 +173,8 @@ validate_field_type(<<"boolean">>) -> <<"boolean">>. +validate_fields(<<"all_fields">>) -> + {ok, all_fields}; validate_fields(Fields) -> try fields_to_json(Fields) of _ -> @@ -174,6 +184,18 @@ validate_fields(Fields) -> end. +validate_ddoc(VProps) -> + try + Def = proplists:get_value(<<"index">>, VProps), + validate_index_def(Def), + Def + catch Error:Reason -> + couch_log:error("Invalid Index Def ~p: Error. ~p, Reason: ~p", + [VProps, Error, Reason]), + invalid_ddoc + end. + + opts() -> [ {<<"default_analyzer">>, [ http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06cf106d/src/mango_idx_view.erl ---------------------------------------------------------------------- diff --git a/src/mango_idx_view.erl b/src/mango_idx_view.erl index cc5bcb3..0cc713f 100644 --- a/src/mango_idx_view.erl +++ b/src/mango_idx_view.erl @@ -14,7 +14,8 @@ -export([ - validate/1, + validate_new/1, + validate_index_def/1, add/2, remove/2, from_ddoc/1, @@ -35,11 +36,15 @@ -include("mango_idx.hrl"). -validate(#idx{}=Idx) -> +validate_new(#idx{}=Idx) -> {ok, Def} = do_validate(Idx#idx.def), {ok, Idx#idx{def=Def}}. +validate_index_def(Def) -> + def_to_json(Def). + + add(#doc{body={Props0}}=DDoc, Idx) -> Views1 = case proplists:get_value(<<"views">>, Props0) of {Views0} -> Views0; @@ -75,17 +80,18 @@ from_ddoc({Props}) -> case lists:keyfind(<<"views">>, 1, Props) of {<<"views">>, {Views}} when is_list(Views) -> lists:flatmap(fun({Name, {VProps}}) -> - Def = proplists:get_value(<<"map">>, VProps), - {Opts0} = proplists:get_value(<<"options">>, VProps), - Opts = lists:keydelete(<<"sort">>, 1, Opts0), - I = #idx{ - type = <<"json">>, - name = Name, - def = Def, - opts = Opts - }, - % TODO: Validate the index definition - [I] + case validate_ddoc(VProps) of + invalid_view -> + []; + {Def, Opts} -> + I = #idx{ + type = <<"json">>, + name = Name, + def = Def, + opts = Opts + }, + [I] + end end, Views); _ -> [] @@ -204,6 +210,20 @@ make_view(Idx) -> {Idx#idx.name, View}. +validate_ddoc(VProps) -> + try + Def = proplists:get_value(<<"map">>, VProps), + validate_index_def(Def), + {Opts0} = proplists:get_value(<<"options">>, VProps), + Opts = lists:keydelete(<<"sort">>, 1, Opts0), + {Def, Opts} + catch Error:Reason -> + couch_log:error("Invalid Index Def ~p. Error: ~p, Reason: ~p", + [VProps, Error, Reason]), + invalid_view + end. + + % This function returns a list of indexes that % can be used to restrict this query. This works by % searching the selector looking for field names that http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06cf106d/src/mango_native_proc.erl ---------------------------------------------------------------------- diff --git a/src/mango_native_proc.erl b/src/mango_native_proc.erl index 9481ca4..822d173 100644 --- a/src/mango_native_proc.erl +++ b/src/mango_native_proc.erl @@ -14,6 +14,9 @@ -behavior(gen_server). +-include("mango_idx.hrl"). + + -export([ start_link/0, set_timeout/2, @@ -72,7 +75,13 @@ handle_call({prompt, [<<"reset">>, _QueryConfig]}, _From, St) -> {reply, true, St#st{indexes=[]}}; handle_call({prompt, [<<"add_fun">>, IndexInfo]}, _From, St) -> - Indexes = St#st.indexes ++ [IndexInfo], + Indexes = case validate_index_info(IndexInfo) of + true -> + St#st.indexes ++ [IndexInfo]; + false -> + couch_log:error("No Valid Indexes For: ~p", [IndexInfo]), + St#st.indexes + end, NewSt = St#st{indexes = Indexes}, {reply, true, NewSt}; @@ -86,7 +95,14 @@ handle_call({prompt, [<<"rereduce">>, _, _]}, _From, St) -> {reply, null, St}; handle_call({prompt, [<<"index_doc">>, Doc]}, _From, St) -> - {reply, index_doc(St, mango_json:to_binary(Doc)), St}; + Vals = case index_doc(St, mango_json:to_binary(Doc)) of + [] -> + [[]]; + Else -> + Else + end, + {reply, Vals, St}; + handle_call(Msg, _From, St) -> {stop, {invalid_call, Msg}, {invalid_call, Msg}, St}. @@ -296,3 +312,21 @@ make_text_field_name([P | Rest], Type) -> Parts = lists:reverse(Rest, [iolist_to_binary([P, ":", Type])]), Escaped = [mango_util:lucene_escape_field(N) || N <- Parts], iolist_to_binary(mango_util:join(".", Escaped)). + + +validate_index_info(IndexInfo) -> + IdxTypes = case module_loaded(dreyfus_index) of + true -> + [mango_idx_view, mango_idx_text]; + false -> + [mango_idx_view] + end, + Results = lists:foldl(fun(IdxType, Results0) -> + try + IdxType:validate_index_def(IndexInfo), + [valid_index | Results0] + catch _:_ -> + [invalid_index | Results0] + end + end, [], IdxTypes), + lists:member(valid_index, Results). \ No newline at end of file http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/06cf106d/test/05-index-selection-test.py ---------------------------------------------------------------------- diff --git a/test/05-index-selection-test.py b/test/05-index-selection-test.py index eecaf17..3a757f8 100644 --- a/test/05-index-selection-test.py +++ b/test/05-index-selection-test.py @@ -74,6 +74,75 @@ class IndexSelectionTests(mango.UserDocsTests): }, use_index=ddocid, explain=True) assert resp["index"]["ddoc"] == ddocid + # This doc will not be saved given the new ddoc validation code + # in couch_mrview + def test_manual_bad_view_idx01(self): + design_doc = { + "_id": "_design/bad_view_index", + "language": "query", + "views": { + "queryidx1": { + "map": { + "fields": { + "age": "asc" + } + }, + "reduce": "_count", + "options": { + "def": { + "fields": [ + { + "age": "asc" + } + ] + }, + "w": 2 + } + } + }, + "views" : { + "views001" : { + "map" : "function(employee){if(employee.training)" + + "{emit(employee.number, employee.training);}}" + } + } + } + with self.assertRaises(KeyError): + self.db.save_doc(design_doc) + + @unittest.skipUnless(mango.has_text_service(), "requires text service") + def test_manual_bad_text_idx(self): + design_doc = { + "_id": "_design/bad_text_index", + "language": "query", + "indexes": { + "text_index": { + "default_analyzer": "keyword", + "default_field": {}, + "selector": {}, + "fields": "all_fields", + "analyzer": { + "name": "perfield", + "default": "keyword", + "fields": { + "$default": "standard" + } + } + } + }, + "indexes": { + "st_index": { + "analyzer": "standard", + "index": "function(doc){\n index(\"st_index\", doc.geometry);\n}" + } + } + } + self.db.save_doc(design_doc) + docs= self.db.find({"age" : 48}) + assert len(docs) == 1 + assert docs[0]["name"]["first"] == "Stephanie" + assert docs[0]["age"] == 48 + @unittest.skipUnless(mango.has_text_service(), "requires text service") class MultiTextIndexSelectionTests(mango.UserDocsTests):