couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dam...@apache.org
Subject svn commit: r763816 - in /couchdb/trunk: share/www/script/test/attachments.js src/couchdb/couch_db.erl src/couchdb/couch_file.erl src/couchdb/couch_stats_collector.erl src/couchdb/couch_stream.erl
Date Thu, 09 Apr 2009 21:37:24 GMT
Author: damien
Date: Thu Apr  9 21:37:23 2009
New Revision: 763816

URL: http://svn.apache.org/viewvc?rev=763816&view=rev
Log:
Fix for attachment sparseness bug COUCHDB-220 by giving each attachment it's own stream and
calling set_min_buffer instead of ensure_buffer. Also fixed spurious couch_file crash messages
by putting the statistics decrement code into a seperate monitoring process.

Modified:
    couchdb/trunk/share/www/script/test/attachments.js
    couchdb/trunk/src/couchdb/couch_db.erl
    couchdb/trunk/src/couchdb/couch_file.erl
    couchdb/trunk/src/couchdb/couch_stats_collector.erl
    couchdb/trunk/src/couchdb/couch_stream.erl

Modified: couchdb/trunk/share/www/script/test/attachments.js
URL: http://svn.apache.org/viewvc/couchdb/trunk/share/www/script/test/attachments.js?rev=763816&r1=763815&r2=763816&view=diff
==============================================================================
--- couchdb/trunk/share/www/script/test/attachments.js (original)
+++ couchdb/trunk/share/www/script/test/attachments.js Thu Apr  9 21:37:23 2009
@@ -142,5 +142,38 @@
   var xhr = CouchDB.request("GET", "/test_suite_db/bin_doc4/attachment.txt");
   T(xhr.status == 200);
   T(xhr.responseText == "This is a string");
+  
+  
+  // Attachment sparseness COUCHDB-220
+
+  var docs = []
+  for (var i = 0; i < 5; i++) {
+      var doc = {
+	  _id: (i).toString(),
+	  _attachments:{
+	      "foo.txt": {
+		  content_type:"text/plain",
+		  data: "VGhpcyBpcyBhIGJhc2U2NCBlbmNvZGVkIHRleHQ="
+	      }
+	  }
+      }
+      docs.push(doc)
+  }
+
+  db.bulkSave(docs);
+  
+  var before = db.info().disk_size;
+
+  // Compact it.
+  T(db.compact().ok);
+  T(db.last_req.status == 202);
+  // compaction isn't instantaneous, loop until done
+  while (db.info().compact_running) {};
+  
+  var after = db.info().disk_size;
+  
+  // Compaction should reduce the database slightly, but not
+  // orders of magnitude (unless attachments introduce sparseness)
+  T(after > before * 0.1, "before: " + before + " after: " + after);
 
 };

Modified: couchdb/trunk/src/couchdb/couch_db.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_db.erl?rev=763816&r1=763815&r2=763816&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_db.erl (original)
+++ couchdb/trunk/src/couchdb/couch_db.erl Thu Apr  9 21:37:23 2009
@@ -548,34 +548,7 @@
 
 
 doc_flush_binaries(Doc, Fd) ->
-    % calc size of binaries to write out
-    Bins = Doc#doc.attachments,
-    PreAllocSize = lists:foldl(
-        fun(BinValue, SizeAcc) ->
-            case BinValue of
-            {_Key, {_Type, {Fd0, _StreamPointer, _Len}}} when Fd0 == Fd ->
-                % already written to our file, nothing to write
-                SizeAcc;
-            {_Key, {_Type, {_OtherFd, _StreamPointer, Len}}} ->
-                % written to a different file
-                SizeAcc + Len;
-            {_Key, {_Type, Bin}} when is_binary(Bin) ->
-                % we have a new binary to write
-                SizeAcc + size(Bin);
-            {_Key, {_Type, {Fun, undefined}}} when is_function(Fun) ->
-                % function without a known length
-                % we'll have to alloc as we go with this one, for now, nothing
-                SizeAcc;
-            {_Key, {_Type, {Fun, Len}}} when is_function(Fun) ->
-                % function to yield binary data with known length
-                SizeAcc + Len
-            end
-        end,
-        0, Bins),
-
-    {ok, OutputStream} = couch_stream:open(Fd),
-    ok = couch_stream:ensure_buffer(OutputStream, PreAllocSize),
-    NewBins = lists:map(
+    NewAttachments = lists:map(
         fun({Key, {Type, BinValue}}) ->
             NewBinValue =
             case BinValue of
@@ -585,6 +558,8 @@
             {OtherFd, StreamPointer, Len} ->
                 % written to a different file (or a closed file
                 % instance, which will cause an error)
+                {ok, OutputStream} = couch_stream:open(Fd),
+                ok = couch_stream:set_min_buffer(OutputStream, Len),
                 {ok, {NewStreamPointer, Len}, _EndSp} =
                 couch_stream:foldl(OtherFd, StreamPointer, Len,
                     fun(Bin, {BeginPointer, SizeAcc}) ->
@@ -597,31 +572,38 @@
                         end
                     end,
                     {{0,0}, 0}),
+                couch_stream:close(OutputStream),
                 {Fd, NewStreamPointer, Len};
             Bin when is_binary(Bin) ->
+                {ok, OutputStream} = couch_stream:open(Fd),
+                ok = couch_stream:set_min_buffer(OutputStream, size(Bin)),
                 {ok, StreamPointer} = couch_stream:write(OutputStream, Bin),
+                couch_stream:close(OutputStream),
                 {Fd, StreamPointer, size(Bin)};
             {StreamFun, undefined} when is_function(StreamFun) ->
                 % max_attachment_chunk_size control the max we buffer in memory
                 MaxChunkSize = list_to_integer(couch_config:get("couchdb", 
                     "max_attachment_chunk_size","4294967296")),
+                {ok, OutputStream} = couch_stream:open(Fd),
                 WriterFun = make_writer_fun(OutputStream),
                 % StreamFun(MaxChunkSize, WriterFun) 
                 % will call our WriterFun
                 % once for each chunk of the attachment.
                 {ok, {TotalLength, NewStreamPointer}} = 
                     StreamFun(MaxChunkSize, WriterFun, {0, nil}),
+                couch_stream:close(OutputStream),
                 {Fd, NewStreamPointer, TotalLength};                
             {Fun, Len} when is_function(Fun) ->
+                {ok, OutputStream} = couch_stream:open(Fd),
+                ok = couch_stream:set_min_buffer(OutputStream, Len),
                 {ok, StreamPointer} =
                         write_streamed_attachment(OutputStream, Fun, Len, nil),
+                couch_stream:close(OutputStream),
                 {Fd, StreamPointer, Len}
             end,
             {Key, {Type, NewBinValue}}
-        end, Bins),
-
-    {ok, _FinalPos} = couch_stream:close(OutputStream),
-    Doc#doc{attachments = NewBins}.
+        end, Doc#doc.attachments),
+    Doc#doc{attachments = NewAttachments}.
 
 
 make_writer_fun(Stream) ->

Modified: couchdb/trunk/src/couchdb/couch_file.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_file.erl?rev=763816&r1=763815&r2=763816&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_file.erl (original)
+++ couchdb/trunk/src/couchdb/couch_file.erl Thu Apr  9 21:37:23 2009
@@ -269,7 +269,6 @@
 % server functions
 
 init({Filepath, Options, ReturnPid, Ref}) ->
-    process_flag(trap_exit, true),
     case lists:member(create, Options) of
     true ->
         filelib:ensure_dir(Filepath),
@@ -285,14 +284,14 @@
                 true ->
                     {ok, 0} = file:position(Fd, 0),
                     ok = file:truncate(Fd),
-                    catch couch_stats_collector:increment({couchdb, open_os_files}),
+                    track_stats(),
                     {ok, Fd};
                 false ->
                     ok = file:close(Fd),
                     init_status_error(ReturnPid, Ref, file_exists)
                 end;
             false ->
-                catch couch_stats_collector:increment({couchdb, open_os_files}),
+                track_stats(),
                 {ok, Fd}
             end;
         Error ->
@@ -304,7 +303,7 @@
         {ok, Fd_Read} ->
             {ok, Fd} = file:open(Filepath, [read, write, raw, binary]),
             ok = file:close(Fd_Read),
-            catch couch_stats_collector:increment({couchdb, open_os_files}),
+            track_stats(),
             {ok, Fd};
         Error ->
             init_status_error(ReturnPid, Ref, Error)
@@ -312,10 +311,21 @@
     end.
 
 
-terminate(_Reason, _Fd) ->
-    catch couch_stats_collector:decrement({couchdb, open_os_files}),
+terminate(_Reason, _Fd) ->         
     ok.
 
+track_stats() ->
+    try couch_stats_collector:increment({couchdb, open_os_files}) of
+    ok ->
+        Self = self(),
+        spawn(
+            fun() ->
+                erlang:monitor(process, Self),
+                receive {'DOWN', _, _, _, _} -> ok end,
+                couch_stats_collector:decrement({couchdb, open_os_files})
+            end)
+    catch _ -> ok
+    end.
 
 handle_call({pread, Pos, Bytes}, _From, Fd) ->
     {reply, file:pread(Fd, Pos, Bytes), Fd};
@@ -349,5 +359,5 @@
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
 
-handle_info({'EXIT', _, Reason}, Fd) ->
-    {stop, Reason, Fd}.
+handle_info(foo, _Fd) ->
+    ok.

Modified: couchdb/trunk/src/couchdb/couch_stats_collector.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_stats_collector.erl?rev=763816&r1=763815&r2=763816&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_stats_collector.erl (original)
+++ couchdb/trunk/src/couchdb/couch_stats_collector.erl Thu Apr  9 21:37:23 2009
@@ -56,13 +56,17 @@
     increment({Module, list_to_atom(integer_to_list(Key))});
 increment(Key) ->
     case catch ets:update_counter(?HIT_COUNTER_TABLE, Key, 1) of
-        {'EXIT', {badarg, _}} -> ets:insert(?HIT_COUNTER_TABLE, {Key, 1});
+        {'EXIT', {badarg, _}} ->
+            true = ets:insert(?HIT_COUNTER_TABLE, {Key, 1}),
+            ok;
         _ -> ok
     end.
     
 decrement(Key) ->
     case catch ets:update_counter(?HIT_COUNTER_TABLE, Key, -1) of
-        {'EXIT', {badarg, _}} -> ets:insert(?HIT_COUNTER_TABLE, {Key, -1});
+        {'EXIT', {badarg, _}} ->
+            true = ets:insert(?HIT_COUNTER_TABLE, {Key, -1}),
+            ok;
         _ -> ok
     end.
     

Modified: couchdb/trunk/src/couchdb/couch_stream.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_stream.erl?rev=763816&r1=763815&r2=763816&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_stream.erl (original)
+++ couchdb/trunk/src/couchdb/couch_stream.erl Thu Apr  9 21:37:23 2009
@@ -38,6 +38,7 @@
     current_pos = 0,
     bytes_remaining = 0,
     next_alloc = 0,
+    first_alloc = 0,
     min_alloc = 16#00010000
     }).
 



Mime
View raw message