couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jch...@apache.org
Subject svn commit: r771466 - in /couchdb/trunk: etc/couchdb/ share/server/ share/www/script/test/ src/couchdb/
Date Mon, 04 May 2009 22:06:01 GMT
Author: jchris
Date: Mon May  4 22:06:01 2009
New Revision: 771466

URL: http://svn.apache.org/viewvc?rev=771466&view=rev
Log:
reduce_limit error is thrown when the reduce function output is not small enough compared
to the input. Errors can be switched off using the config API.

Modified:
    couchdb/trunk/etc/couchdb/default.ini.tpl.in
    couchdb/trunk/share/server/loop.js
    couchdb/trunk/share/server/state.js
    couchdb/trunk/share/server/views.js
    couchdb/trunk/share/www/script/test/design_docs.js
    couchdb/trunk/share/www/script/test/view_errors.js
    couchdb/trunk/src/couchdb/couch_query_servers.erl
    couchdb/trunk/src/couchdb/couch_view_group.erl

Modified: couchdb/trunk/etc/couchdb/default.ini.tpl.in
URL: http://svn.apache.org/viewvc/couchdb/trunk/etc/couchdb/default.ini.tpl.in?rev=771466&r1=771465&r2=771466&view=diff
==============================================================================
--- couchdb/trunk/etc/couchdb/default.ini.tpl.in (original)
+++ couchdb/trunk/etc/couchdb/default.ini.tpl.in Mon May  4 22:06:01 2009
@@ -26,6 +26,9 @@
 [query_servers]
 javascript = %bindir%/%couchjs_command_name% %localdatadir%/server/main.js
 
+[query_server_config]
+reduce_limit = true
+
 ; enable external as an httpd handler, then link it with commands here.
 ; note, this api is still under consideration.
 ; [external]

Modified: couchdb/trunk/share/server/loop.js
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/server/loop.js?rev=771466&r1=771465&r2=771466&view=diff
==============================================================================
--- couchdb/trunk/share/server/loop.js (original)
+++ couchdb/trunk/share/server/loop.js Mon May  4 22:06:01 2009
@@ -28,7 +28,7 @@
 //
 // Responses are json values followed by a new line ("\n")
 
-var cmd, cmdkey;
+var line, cmd, cmdkey;
 
 var dispatch = {
   "reset"      : State.reset,
@@ -43,7 +43,9 @@
   "list_tail"  : Render.listTail 
 };
 
-while (cmd = eval(readline())) {
+while (line = eval(readline())) {
+  cmd = eval(line)
+  line_length = line.length
   try {
     cmdkey = cmd.shift();
     if (dispatch[cmdkey]) {

Modified: couchdb/trunk/share/server/state.js
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/server/state.js?rev=771466&r1=771465&r2=771466&view=diff
==============================================================================
--- couchdb/trunk/share/server/state.js (original)
+++ couchdb/trunk/share/server/state.js Mon May  4 22:06:01 2009
@@ -13,12 +13,14 @@
 // globals used by other modules and functions
 var funs = [];        // holds functions used for computation
 var funsrc = [];      // holds function source for debug info
+var query_config = {};
 var State = (function() {
   return {
-    reset : function() {
+    reset : function(config) {
       // clear the globals and run gc
       funs = [];
       funsrc = [];
+      query_config = config;
       gc();
       print("true"); // indicates success
     },

Modified: couchdb/trunk/share/server/views.js
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/server/views.js?rev=771466&r1=771465&r2=771466&view=diff
==============================================================================
--- couchdb/trunk/share/server/views.js (original)
+++ couchdb/trunk/share/server/views.js Mon May  4 22:06:01 2009
@@ -46,7 +46,17 @@
         reductions[i] = null;
       }
     }
-    print("[true," + toJSON(reductions) + "]");
+    var reduce_line = toJSON(reductions);
+    var reduce_length = reduce_line.length;
+    if (query_config && query_config.reduce_limit &&
+          reduce_length > 200 && ((reduce_length * 2) > line.length)) {
+      throw {
+        error:"reduce_overflow_error",
+        reason: "Reduce output must shrink more rapidly. Current output: '"+reduce_line+"'"
+      };
+    } else {
+      print("[true," + reduce_line + "]");
+    }
   };
   
   return {

Modified: couchdb/trunk/share/www/script/test/design_docs.js
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/www/script/test/design_docs.js?rev=771466&r1=771465&r2=771466&view=diff
==============================================================================
--- couchdb/trunk/share/www/script/test/design_docs.js (original)
+++ couchdb/trunk/share/www/script/test/design_docs.js Mon May  4 22:06:01 2009
@@ -16,6 +16,12 @@
   db.createDb();
   if (debug) debugger;
 
+  run_on_modified_server(
+    [{section: "query_server_config",
+      key: "reduce_limit",
+      value: "false"}],
+function() {
+
   var numDocs = 500;
 
   function makebigstring(power) {
@@ -104,4 +110,5 @@
   restartServer();
   T(db.open(designDoc._id) == null);
   T(db.view("test/no_docs") == null);
+});
 };

Modified: couchdb/trunk/share/www/script/test/view_errors.js
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/www/script/test/view_errors.js?rev=771466&r1=771465&r2=771466&view=diff
==============================================================================
--- couchdb/trunk/share/www/script/test/view_errors.js (original)
+++ couchdb/trunk/share/www/script/test/view_errors.js Mon May  4 22:06:01 2009
@@ -75,13 +75,26 @@
     _id:"_design/test",
     language: "javascript",
     views: {
-      no_reduce: {map:"function(doc) {emit(doc._id, null);}"},
-      with_reduce: {map:"function (doc) {emit(doc.integer, doc.integer)};",
-                reduce:"function (keys, values) { return sum(values); };"},
+      "no_reduce": {map:"function(doc) {emit(doc._id, null);}"},
+      "with_reduce": {
+        map:"function (doc) {emit(doc.integer, doc.integer)};",
+        reduce:"function (keys, values) { return sum(values); };"}
     }
   };
   T(db.save(designDoc).ok);
   
+  var designDoc2 = {
+    _id:"_design/testbig",
+    language: "javascript",
+    views: {
+      "reduce_too_big"  : {
+        map:"function (doc) {emit(doc.integer, doc.integer)};",
+        reduce:"function (keys, values) { var chars = []; for (var i=0; i < 1000; i++)
{chars.push('wazzap');};return chars; };"}
+    }
+  };
+  T(db.save(designDoc2).ok);
+  
+  
   try {
       db.view("test/no_reduce", {group: true});
       T(0 == 1);
@@ -117,4 +130,11 @@
   result = JSON.parse(xhr.responseText);
   T(result.error == "bad_request");
   T(result.reason == "`keys` member must be a array.");
+
+  // if the reduce grows to fast, throw an overflow error
+  var path = "/test_suite_db/_design/testbig/_view/reduce_too_big";
+  xhr = CouchDB.request("GET", path);
+  T(xhr.status == 500);
+  result = JSON.parse(xhr.responseText);
+  T(result.error == "reduce_overflow_error");
 };

Modified: couchdb/trunk/src/couchdb/couch_query_servers.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_query_servers.erl?rev=771466&r1=771465&r2=771466&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_query_servers.erl (original)
+++ couchdb/trunk/src/couchdb/couch_query_servers.erl Mon May  4 22:06:01 2009
@@ -85,23 +85,27 @@
 rereduce(Lang, RedSrcs, ReducedValues) ->
     Pid = get_os_process(Lang),
     Grouped = group_reductions_results(ReducedValues),
-    Results = lists:zipwith(
+    Results = try lists:zipwith(
         fun(FunSrc, Values) ->
             [true, [Result]] = 
                 couch_os_process:prompt(Pid, [<<"rereduce">>, [FunSrc], Values]),
             Result
-        end, RedSrcs, Grouped),
-        
-    ok = ret_os_process(Lang, Pid),
+        end, RedSrcs, Grouped)
+    after
+        ok = ret_os_process(Lang, Pid)
+    end,
     {ok, Results}.
 
 reduce(_Lang, [], _KVs) ->
     {ok, []};
 reduce(Lang, RedSrcs, KVs) ->
     Pid = get_os_process(Lang),
-    [true, Results] = couch_os_process:prompt(Pid, 
-            [<<"reduce">>, RedSrcs, KVs]),
-    ok = ret_os_process(Lang, Pid),
+    Results = try couch_os_process:prompt(Pid, 
+            [<<"reduce">>, RedSrcs, KVs]) of
+        [true, Reductions] -> Reductions
+    after
+        ok = ret_os_process(Lang, Pid)
+    end,
     {ok, Results}.
 
 validate_doc_update(Lang, FunSrc, EditDoc, DiskDoc, Ctx) ->
@@ -209,7 +213,8 @@
         add_value(PidLangs, Pid, Lang),
         rem_from_list(Pids, Lang, Pid),
         add_to_list(InUse, Lang, Pid),
-        true = couch_os_process:prompt(Pid, [<<"reset">>]),
+        QueryConfig = get_query_server_config(),
+        true = couch_os_process:prompt(Pid, [<<"reset">>, QueryConfig]),
         {reply, Pid, Server};
     _ ->
         {ok, Pid} = new_process(Langs, Lang),
@@ -249,6 +254,11 @@
 
 % Private API
 
+get_query_server_config() ->
+    ReduceLimit = list_to_atom(
+        couch_config:get("query_server_config","reduce_limit","true")),
+    {[{<<"reduce_limit">>, ReduceLimit}]}.
+
 new_process(Langs, Lang) ->
     Proc =
     case ets:lookup(Langs, Lang) of

Modified: couchdb/trunk/src/couchdb/couch_view_group.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_view_group.erl?rev=771466&r1=771465&r2=771466&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_view_group.erl (original)
+++ couchdb/trunk/src/couchdb/couch_view_group.erl Mon May  4 22:06:01 2009
@@ -42,9 +42,9 @@
     {ok, Group, RefCounter} ->
         couch_ref_counter:add(RefCounter),
         {ok, Group};
-    Else ->
-        ?LOG_DEBUG("get_updated_group replied with _Else ~p", [Else]),
-        Else
+    Error ->
+        ?LOG_DEBUG("request_group Error ~p", [Error]),
+        throw(Error)
     end.
 
 
@@ -261,6 +261,10 @@
 handle_info({'EXIT', _FromPid, normal}, State) ->
     {noreply, State};
     
+handle_info({'EXIT', FromPid, {{nocatch, Reason}, Trace}}, State) ->
+    ?LOG_DEBUG("Uncaught throw() in linked pid: ~p", [{FromPid, Reason}]),
+    {stop, Reason, State};
+
 handle_info({'EXIT', FromPid, Reason}, State) ->
     ?LOG_DEBUG("Exit from linked pid: ~p", [{FromPid, Reason}]),
     {stop, Reason, State};



Mime
View raw message