couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tonysu...@apache.org
Subject couchdb-mango git commit: Add config parameter to reject index all text indexes
Date Tue, 06 Dec 2016 21:24:26 GMT
Repository: couchdb-mango
Updated Branches:
  refs/heads/master 50066bc84 -> 4afd60e84


Add config parameter to reject index all text indexes

Text indexes that index all fields can sometimes lead to OOM issues
when users have documents with nested array fields. This change adds
in a config parameter to provide the ability to log users who do this
and also reject new requests. Note that we need to pass in a db
record to validate_new because it contains user and db name info that
will be logged.

COUCHDB-3249


Project: http://git-wip-us.apache.org/repos/asf/couchdb-mango/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mango/commit/4afd60e8
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mango/tree/4afd60e8
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mango/diff/4afd60e8

Branch: refs/heads/master
Commit: 4afd60e84d0e1c57f5d6a1e3542955faa565ca4b
Parents: 50066bc
Author: Tony Sun <tony.sun@cloudant.com>
Authored: Tue Dec 6 00:46:51 2016 -0800
Committer: Tony Sun <tony.sun@cloudant.com>
Committed: Tue Dec 6 12:59:55 2016 -0800

----------------------------------------------------------------------
 src/mango_error.erl    |  6 ++++
 src/mango_httpd.erl    |  2 +-
 src/mango_idx.erl      |  6 ++--
 src/mango_idx_text.erl | 83 +++++++++++++++++++++++++++++++++++++++++++--
 src/mango_idx_view.erl |  4 +--
 5 files changed, 93 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/4afd60e8/src/mango_error.erl
----------------------------------------------------------------------
diff --git a/src/mango_error.erl b/src/mango_error.erl
index f0a8ee2..76a8362 100644
--- a/src/mango_error.erl
+++ b/src/mango_error.erl
@@ -174,6 +174,12 @@ info(mango_idx_text, {index_not_found, BadIdx}) ->
         <<"index_not_found">>,
         fmt("Text index ~s not found in this design doc.", [BadIdx])
     };
+info(mango_idx_text, index_all_disabled) ->
+    {
+        403,
+        <<"index_all_disabled">>,
+        <<"New text indexes are forbidden to index all fields.">>
+    };
 
 info(mango_opts, {invalid_bulk_docs, Val}) ->
     {

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/4afd60e8/src/mango_httpd.erl
----------------------------------------------------------------------
diff --git a/src/mango_httpd.erl b/src/mango_httpd.erl
index bde3850..a088276 100644
--- a/src/mango_httpd.erl
+++ b/src/mango_httpd.erl
@@ -84,7 +84,7 @@ handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) ->
     chttpd:validate_ctype(Req, "application/json"),
     {ok, Opts} = mango_opts:validate_idx_create(chttpd:json_body_obj(Req)),
     {ok, Idx0} = mango_idx:new(Db, Opts),
-    {ok, Idx} = mango_idx:validate_new(Idx0),
+    {ok, Idx} = mango_idx:validate_new(Idx0, Db),
     {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx)),
     Id = Idx#idx.ddoc,
     Name = Idx#idx.name,

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/4afd60e8/src/mango_idx.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx.erl b/src/mango_idx.erl
index 11c713b..bc88b97 100644
--- a/src/mango_idx.erl
+++ b/src/mango_idx.erl
@@ -23,7 +23,7 @@
     for_sort/2,
 
     new/2,
-    validate_new/1,
+    validate_new/2,
     add/2,
     remove/2,
     from_ddoc/2,
@@ -136,9 +136,9 @@ new(Db, Opts) ->
     }}.
 
 
-validate_new(Idx) ->
+validate_new(Idx, Db) ->
     Mod = idx_mod(Idx),
-    Mod:validate_new(Idx).
+    Mod:validate_new(Idx, Db).
 
 
 add(DDoc, Idx) ->

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/4afd60e8/src/mango_idx_text.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx_text.erl b/src/mango_idx_text.erl
index 40a1fd3..4cfda5a 100644
--- a/src/mango_idx_text.erl
+++ b/src/mango_idx_text.erl
@@ -14,7 +14,7 @@
 
 
 -export([
-    validate_new/1,
+    validate_new/2,
     validate_fields/1,
     validate_index_def/1,
     add/2,
@@ -32,8 +32,9 @@
 -include("mango_idx.hrl").
 
 
-validate_new(#idx{}=Idx) ->
+validate_new(#idx{}=Idx, Db) ->
     {ok, Def} = do_validate(Idx#idx.def),
+    maybe_reject_index_all_req(Def, Db),
     {ok, Idx#idx{def=Def}}.
 
 
@@ -327,3 +328,81 @@ indexable_fields(Fields, {op_null, {_, _}}) ->
 
 indexable_fields(Fields, {op_default, _}) ->
     [<<"$default">> | Fields].
+
+
+maybe_reject_index_all_req({Def}, #db{name=DbName, user_ctx=Ctx}) ->
+    User = Ctx#user_ctx.name,
+    Fields = couch_util:get_value(fields, Def),
+    case {Fields, forbid_index_all()} of
+        {all_fields, "true"} ->
+            ?MANGO_ERROR(index_all_disabled);
+        {all_fields, "warn"} ->
+            couch_log:warning("User ~p is indexing all fields in db ~p",
+                [User, DbName]);
+        _ ->
+            ok
+    end.
+
+
+forbid_index_all() ->
+    config:get("mango", "index_all_disabled", "false").
+
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+
+setup() ->
+    test_util:start_couch(),
+    meck:expect(couch_log, warning, 2,
+        fun(_,_) ->
+            throw({test_error, logged_warning})
+        end),
+    %default index all def that generates {fields, all_fields}
+    Index = #idx{def={[]}},
+    Db = #db{name = <<"testdb">>, user_ctx=#user_ctx{name = <<"u1">>}},
+    {Index, Db}.
+
+
+teardown(_) ->
+    ok = config:delete("mango", "index_all_disabled"),
+    test_util:stop_couch().
+
+
+index_all_test_() ->
+    {
+        foreach,
+        fun setup/0,
+        fun teardown/1,
+        [
+            fun forbid_index_all/1,
+            fun default_and_false_index_all/1,
+            fun warn_index_all/1
+        ]
+
+    }.
+
+
+forbid_index_all({Idx, Db}) ->
+    ok = config:set("mango", "index_all_disabled", "true"),
+    ?_assertThrow({mango_error, ?MODULE, index_all_disabled},
+        validate_new(Idx, Db)
+    ).
+
+
+default_and_false_index_all({Idx, Db}) ->
+    {ok, #idx{def={Def}}} = validate_new(Idx, Db),
+    Fields = couch_util:get_value(fields, Def),
+    ?_assertEqual(all_fields, Fields),
+    ok = config:set("mango", "index_all_disabled", "false"),
+    {ok, #idx{def={Def2}}} = validate_new(Idx, Db),
+    Fields2 = couch_util:get_value(fields, Def2),
+    ?_assertEqual(all_fields, Fields2).
+
+
+warn_index_all({Idx, Db}) ->
+    ok = config:set("mango", "index_all_disabled", "warn"),
+    ?_assertThrow({test_error, logged_warning}, validate_new(Idx, Db)).
+
+
+-endif.

http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/4afd60e8/src/mango_idx_view.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx_view.erl b/src/mango_idx_view.erl
index 0cc713f..8bad34c 100644
--- a/src/mango_idx_view.erl
+++ b/src/mango_idx_view.erl
@@ -14,7 +14,7 @@
 
 
 -export([
-    validate_new/1,
+    validate_new/2,
     validate_index_def/1,
     add/2,
     remove/2,
@@ -36,7 +36,7 @@
 -include("mango_idx.hrl").
 
 
-validate_new(#idx{}=Idx) ->
+validate_new(#idx{}=Idx, _Db) ->
     {ok, Def} = do_validate(Idx#idx.def),
     {ok, Idx#idx{def=Def}}.
 


Mime
View raw message