couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tonysu...@apache.org
Subject chttpd commit: updated refs/heads/2992-limit-doc-size to f7affd0 [Forced Update!]
Date Sun, 17 Apr 2016 21:42:59 GMT
Repository: couchdb-chttpd
Updated Branches:
  refs/heads/2992-limit-doc-size 5181ec865 -> f7affd0d2 (forced update)


Add support for single_max_doc

This adds support for a new configuration param single_max_doc_size.
Single documents that exceed this document will receive a too_large
error.

COUCHDB-2992


Project: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/commit/f7affd0d
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/tree/f7affd0d
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/diff/f7affd0d

Branch: refs/heads/2992-limit-doc-size
Commit: f7affd0d276e12b29e930666dd36ff7b9fdd8e5d
Parents: be1e959
Author: Tony Sun <tony.sun@cloudant.com>
Authored: Sat Apr 16 19:42:46 2016 -0700
Committer: Tony Sun <tony.sun@cloudant.com>
Committed: Sun Apr 17 14:40:40 2016 -0700

----------------------------------------------------------------------
 src/chttpd.erl                   |  4 ++
 src/chttpd_db.erl                | 82 +++++++++++++++++++++---------
 test/chttpd_db_doc_size_test.erl | 96 +++++++++++++++++++++++++++++++++++
 3 files changed, 158 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/f7affd0d/src/chttpd.erl
----------------------------------------------------------------------
diff --git a/src/chttpd.erl b/src/chttpd.erl
index ee90e5d..1f8c160 100644
--- a/src/chttpd.erl
+++ b/src/chttpd.erl
@@ -858,6 +858,10 @@ error_info({error, {database_name_too_long, DbName}}) ->
         <<"At least one path segment of `", DbName/binary, "` is too long.">>};
 error_info({missing_stub, Reason}) ->
     {412, <<"missing_stub">>, Reason};
+error_info({single_doc_too_large, MaxSize}) ->
+    SizeBin = list_to_binary(integer_to_list(MaxSize)),
+    {413, <<"too_large">>, <<"Document exceeded single_max_doc_size:",
+        SizeBin/binary>>};
 error_info(request_entity_too_large) ->
     {413, <<"too_large">>, <<"the request entity is too large">>};
 error_info({error, security_migration_updates_disabled}) ->

http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/f7affd0d/src/chttpd_db.erl
----------------------------------------------------------------------
diff --git a/src/chttpd_db.erl b/src/chttpd_db.erl
index 9347c1a..2a3fe1b 100644
--- a/src/chttpd_db.erl
+++ b/src/chttpd_db.erl
@@ -308,7 +308,10 @@ db_req(#httpd{method='POST', path_parts=[DbName], user_ctx=Ctx}=Req,
Db) ->
     W = chttpd:qs_value(Req, "w", integer_to_list(mem3:quorum(Db))),
     Options = [{user_ctx,Ctx}, {w,W}],
 
-    Doc = couch_doc:from_json_obj(chttpd:json_body(Req)),
+    Body = chttpd:json_body(Req),
+    ok = verify_doc_size(Body),
+
+    Doc = couch_doc:from_json_obj(Body),
     Doc2 = case Doc#doc.id of
         <<"">> ->
             Doc#doc{id=couch_uuids:new(), revs={0, []}};
@@ -374,6 +377,7 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>],
user_ctx=Ctx}=Req,
     DocsArray0 ->
         DocsArray0
     end,
+    {ExceedErrs, DocsArray1} = remove_exceed_docs(DocsArray),
     couch_stats:update_histogram([couchdb, httpd, bulk_docs], length(DocsArray)),
     W = case couch_util:get_value(<<"w">>, JsonProps) of
     Value when is_integer(Value) ->
@@ -401,39 +405,38 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>],
user_ctx=Ctx}=Req,
                 end,
                 Doc#doc{id=Id}
             end,
-            DocsArray),
+            DocsArray1),
         Options2 =
         case couch_util:get_value(<<"all_or_nothing">>, JsonProps) of
         true  -> [all_or_nothing|Options];
         _ -> Options
         end,
-        case fabric:update_docs(Db, Docs, Options2) of
-        {ok, Results} ->
+        {Code, Results} = case fabric:update_docs(Db, Docs, Options2) of
+        {ok, Results0} ->
             % output the results
-            DocResults = lists:zipwith(fun update_doc_result_to_json/2,
-                Docs, Results),
-            send_json(Req, 201, DocResults);
-        {accepted, Results} ->
+            {201, lists:zipwith(fun update_doc_result_to_json/2, Docs,
+                Results0)};
+        {accepted, Results0} ->
             % output the results
-            DocResults = lists:zipwith(fun update_doc_result_to_json/2,
-                Docs, Results),
-            send_json(Req, 202, DocResults);
+            {202, lists:zipwith(fun update_doc_result_to_json/2, Docs,
+                Results0)};
         {aborted, Errors} ->
-            ErrorsJson =
-                lists:map(fun update_doc_result_to_json/1, Errors),
-            send_json(Req, 417, ErrorsJson)
-        end;
+            {417, lists:map(fun update_doc_result_to_json/1, Errors)}
+        end,
+        FinalResults =  lists:keysort(1, ExceedErrs ++ Results),
+        send_json(Req, Code, FinalResults);
     false ->
         Docs = [couch_doc:from_json_obj(JsonObj) || JsonObj <- DocsArray],
         [validate_attachment_names(D) || D <- Docs],
-        case fabric:update_docs(Db, Docs, [replicated_changes|Options]) of
-        {ok, Errors} ->
-            ErrorsJson = lists:map(fun update_doc_result_to_json/1, Errors),
-            send_json(Req, 201, ErrorsJson);
-        {accepted, Errors} ->
-            ErrorsJson = lists:map(fun update_doc_result_to_json/1, Errors),
-            send_json(Req, 202, ErrorsJson)
-        end
+        {Code, Errs1} = case fabric:update_docs(Db, Docs, [replicated_changes|Options]) of
+        {ok, Errs0} ->
+            {201, Errs0};
+        {accepted, Errs0} ->
+            {202, Errs0}
+        end,
+        Errs2 = lists:map(fun update_doc_result_to_json/1, Errs1),
+        FinalErrors = lists:keysort(1, ExceedErrs ++ Errs2),
+        send_json(Req, Code, FinalErrors)
     end;
 
 db_req(#httpd{path_parts=[_,<<"_bulk_docs">>]}=Req, _Db) ->
@@ -765,7 +768,9 @@ db_doc_req(#httpd{method='PUT', user_ctx=Ctx}=Req, Db, DocId) ->
         case chttpd:qs_value(Req, "batch") of
         "ok" ->
             % batch
-            Doc = couch_doc_from_req(Req, DocId, chttpd:json_body(Req)),
+            Body = chttpd:json_body(Req),
+            ok = verify_doc_size(Body),
+            Doc = couch_doc_from_req(Req, DocId, Body),
 
             spawn(fun() ->
                     case catch(fabric:update_doc(Db, Doc, Options)) of
@@ -782,6 +787,7 @@ db_doc_req(#httpd{method='PUT', user_ctx=Ctx}=Req, Db, DocId) ->
         _Normal ->
             % normal
             Body = chttpd:json_body(Req),
+            ok = verify_doc_size(Body),
             Doc = couch_doc_from_req(Req, DocId, Body),
             send_updated_doc(Req, Db, DocId, Doc, RespHeaders, UpdateType)
         end
@@ -1628,6 +1634,34 @@ bulk_get_open_doc_revs1(Db, Props, _, {DocId, Revs, Options}) ->
             {DocId, Results, Options}
     end.
 
+remove_exceed_docs(DocArray0) ->
+    MaxSize = list_to_integer(config:get("couchdb",
+        "single_max_doc_size", "1048576")),
+    {ExceedDocs, DocsArray} = lists:splitwith(fun (Doc) ->
+        exceed_doc_size(Doc, MaxSize)
+    end, DocArray0),
+    ExceedErrs = lists:map (fun ({Doc}) ->
+        DocId = case couch_util:get_value(<<"_id">>, Doc) of
+            undefined -> couch_uuids:new();
+            Id0 -> Id0
+        end,
+        Reason = lists:concat(["Document exceeded single_max_doc_size:",
+            " of ", MaxSize, " bytes"]),
+        {[{id, DocId}, {error, <<"too_large">>},
+            {reason, ?l2b(Reason)}]}
+    end, ExceedDocs),
+    {ExceedErrs, DocsArray}.
+
+exceed_doc_size(JsonBody, MaxSize) ->
+    size(term_to_binary(JsonBody)) > MaxSize.
+
+verify_doc_size(JsonBody) ->
+    SMaxSize = list_to_integer(config:get("couchdb",
+        "single_max_doc_size", "1048576")),
+    case exceed_doc_size(JsonBody, SMaxSize) of
+        true -> throw({single_doc_too_large, SMaxSize});
+        false -> ok
+    end.
 
 parse_field(<<"id">>, undefined) ->
     {ok, undefined};

http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/f7affd0d/test/chttpd_db_doc_size_test.erl
----------------------------------------------------------------------
diff --git a/test/chttpd_db_doc_size_test.erl b/test/chttpd_db_doc_size_test.erl
new file mode 100644
index 0000000..02ae3cd
--- /dev/null
+++ b/test/chttpd_db_doc_size_test.erl
@@ -0,0 +1,96 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(chttpd_db_doc_size_test).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-define(USER, "chttpd_db_test_admin").
+-define(PASS, "pass").
+-define(AUTH, {basic_auth, {?USER, ?PASS}}).
+-define(CONTENT_JSON, {"Content-Type", "application/json"}).
+
+setup() ->
+    ok = config:set("admins", ?USER, ?PASS, _Persist=false),
+    ok = config:set("couchdb", "single_max_doc_size", "50"),
+    TmpDb = ?tempdb(),
+    Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
+    Port = mochiweb_socket_server:get(chttpd, port),
+    Url = lists:concat(["http://", Addr, ":", Port, "/", ?b2l(TmpDb)]),
+    create_db(Url),
+    Url.
+
+teardown(Url) ->
+    delete_db(Url),
+    ok = config:delete("admins", ?USER, _Persist=false),
+    ok = config:delete("couchdb", "single_max_doc_size").
+
+create_db(Url) ->
+    {ok, Status, _, _} = test_request:put(Url, [?CONTENT_JSON, ?AUTH], "{}"),
+    ?assert(Status =:= 201 orelse Status =:= 202).
+
+delete_db(Url) ->
+    {ok, 200, _, _} = test_request:delete(Url, [?AUTH]).
+
+all_test_() ->
+    {
+        "chttpd db single doc max size test",
+        {
+            setup,
+            fun chttpd_test_util:start_couch/0, fun chttpd_test_util:stop_couch/1,
+            {
+                foreach,
+                fun setup/0, fun teardown/1,
+                [
+                    fun post_single_doc/1,
+                    fun put_single_doc/1,
+                    fun bulk_doc/1
+                ]
+            }
+        }
+    }.
+
+post_single_doc(Url) ->
+    ?_assertEqual({<<"error">>, <<"too_large">>},
+        begin
+            NewDoc = "{\"post_single_doc\": \"some_doc\",
+                \"_id\": \"testdoc\", \"should_be\" : \"too_large\"}",
+            {ok, _, _, ResultBody} = test_request:post(Url,
+                [?CONTENT_JSON, ?AUTH], NewDoc),
+            {ErrorMsg} = ?JSON_DECODE(ResultBody),
+            lists:nth(1, ErrorMsg)
+        end).
+
+put_single_doc(Url) ->
+    ?_assertEqual({<<"error">>, <<"too_large">>},
+        begin
+            NewDoc = "{\"post_single_doc\": \"some_doc\",
+                \"_id\": \"testdoc\", \"should_be\" : \"too_large\"}",
+            {ok, _, _, ResultBody} = test_request:put(Url ++ "/" ++ "testid",
+                [?CONTENT_JSON, ?AUTH], NewDoc),
+            {ErrorMsg} = ?JSON_DECODE(ResultBody),
+            lists:nth(1, ErrorMsg)
+        end).
+
+bulk_doc(Url) ->
+    NewDoc = "{\"docs\": [{\"doc1\": 1}, {\"errordoc\":
+        \"this_should_be_the_error_document\"}]}",
+    {ok, _, _, ResultBody} = test_request:post(Url ++ "/_bulk_docs/",
+        [?CONTENT_JSON, ?AUTH], NewDoc),
+    ResultJson = ?JSON_DECODE(ResultBody),
+    {InnerJson1} = lists:nth(1, ResultJson),
+    {InnerJson2} = lists:nth(2, ResultJson),
+    Error = couch_util:get_value(<<"error">>, InnerJson1),
+    Msg = couch_util:get_value(<<"error">>, InnerJson2),
+    ?_assertEqual(<<"too_large">>, Error),
+    ?_assertEqual(undefined, Msg).


Mime
View raw message