Author: fdmanana
Date: Tue Nov 16 13:42:28 2010
New Revision: 1035636

URL: http://svn.apache.org/viewvc?rev=1035636&view=rev
Log:
New replicator: replicate attachments in compressed form (if they are stored in compressed form).

Modified:
    couchdb/branches/new_replicator/share/www/script/test/new_replication.js
    couchdb/branches/new_replicator/src/couchdb/couch_api_wrap.erl
    couchdb/branches/new_replicator/src/couchdb/couch_httpd_db.erl

Modified: couchdb/branches/new_replicator/share/www/script/test/new_replication.js
URL: http://svn.apache.org/viewvc/couchdb/branches/new_replicator/share/www/script/test/new_replication.js?rev=1035636&r1=1035635&r2=1035636&view=diff
==============================================================================
--- couchdb/branches/new_replicator/share/www/script/test/new_replication.js (original)
+++ couchdb/branches/new_replicator/share/www/script/test/new_replication.js Tue Nov 16 13:42:28 2010
@@ -49,6 +49,51 @@ couchTests.new_replication = function(de
   var i, j, k;
 
 
+  function makeAttData(minSize) {
+    var data = att1_data;
+
+    while (data.length < minSize) {
+      data = data + att1_data;
+    }
+    return data;
+  }
+
+
+  function enableAttCompression(level, types) {
+    var xhr = CouchDB.request(
+      "PUT",
+      "/_config/attachments/compression_level",
+      {
+        body: JSON.stringify(level),
+        headers: {"X-Couch-Persist": "false"}
+      }
+    );
+    T(xhr.status === 200);
+    xhr = CouchDB.request(
+      "PUT",
+      "/_config/attachments/compressible_types",
+      {
+        body: JSON.stringify(types),
+        headers: {"X-Couch-Persist": "false"}
+      }
+    );
+    T(xhr.status === 200);
+  }
+
+
+  function disableAttCompression() {
+    var xhr = CouchDB.request(
+      "PUT",
+      "/_config/attachments/compression_level",
+      {
+        body: JSON.stringify("0"),
+        headers: {"X-Couch-Persist": "false"}
+      }
+    );
+    T(xhr.status === 200);
+  }
+
+
   function populateDb(db, docs, dontRecreateDb) {
     if (dontRecreateDb !== true) {
       db.deleteDb();
@@ -1090,6 +1135,67 @@ couchTests.new_replication = function(de
 
 
   //
+  // test replication of compressed attachments
+  //
+  doc = {
+    _id: "foobar"
+  };
+  var bigTextAtt = makeAttData(128 * 1024);
+  var attName = "readme.txt";
+  var xhr = CouchDB.request("GET", "/_config/attachments/compression_level");
+  var compressionLevel = JSON.parse(xhr.responseText);
+  xhr = CouchDB.request("GET", "/_config/attachments/compressible_types");
+  var compressibleTypes = JSON.parse(xhr.responseText);
+
+  for (i = 0; i < dbPairs.length; i++) {
+    populateDb(sourceDb, [doc]);
+    populateDb(targetDb, []);
+
+    // enable compression of text types
+    enableAttCompression("8", "text/*");
+
+    // add text attachment to foobar doc
+    xhr = CouchDB.request(
+      "PUT",
+      "/" + sourceDb.name + "/" + doc._id + "/" + attName + "?rev=" + doc._rev,
+      {
+        body: bigTextAtt,
+        headers: {"Content-Type": "text/plain"}
+      }
+    );
+    T(xhr.status === 201);
+
+    // disable compression and replicate
+    disableAttCompression();
+
+    repResult = CouchDB.new_replicate(dbPairs[i].source, dbPairs[i].target);
+    T(repResult.ok === true);
+    T(repResult.history instanceof Array);
+    T(repResult.history.length === 1);
+    T(repResult.history[0].missing_checked === 1);
+    T(repResult.history[0].missing_found === 1);
+    T(repResult.history[0].docs_read === 1);
+    T(repResult.history[0].docs_written === 1);
+    T(repResult.history[0].doc_write_failures === 0);
+
+    copy = targetDb.open(
+      doc._id,
+      {att_encoding_info: true, bypass_cache: Math.round(Math.random() * 1000)}
+    );
+    T(copy !== null);
+    T(attName in copy._attachments);
+    TEquals("gzip", copy._attachments[attName].encoding);
+    TEquals("number", typeof copy._attachments[attName].length);
+    TEquals("number", typeof copy._attachments[attName].encoded_length);
+    T(copy._attachments[attName].encoded_length < copy._attachments[attName].length);
+  }
+
+  delete bigTextAtt;
+  // restore original settings
+  enableAttCompression(compressionLevel, compressibleTypes);
+
+
+  //
   // test replication triggered by non admins
   //
 

Modified: couchdb/branches/new_replicator/src/couchdb/couch_api_wrap.erl
URL: http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_api_wrap.erl?rev=1035636&r1=1035635&r2=1035636&view=diff
==============================================================================
--- couchdb/branches/new_replicator/src/couchdb/couch_api_wrap.erl (original)
+++ couchdb/branches/new_replicator/src/couchdb/couch_api_wrap.erl Tue Nov 16 13:42:28 2010
@@ -151,7 +151,7 @@ get_missing_revs(Db, IdRevs) ->
 open_doc_revs(#httpdb{} = HttpDb, Id, Revs, Options, Fun, Acc) ->
     Path = encode_doc_id(Id),
     QArgs = options_to_query_args(
-        HttpDb, Path, [revs, {open_revs, Revs} | Options]),
+        HttpDb, Path, [revs, {open_revs, Revs}, att_encoding_info | Options]),
     Self = self(),
     Streamer = spawn_link(fun() ->
             send_req(
@@ -213,9 +213,10 @@ update_doc(#httpdb{} = HttpDb, #doc{id =
     end ++ options_to_query_args(Options, []),
     Boundary = couch_uuids:random(),
     JsonBytes = ?JSON_ENCODE(
-        couch_doc:to_json_obj(Doc, [revs, attachments, follows | Options])),
+        couch_doc:to_json_obj(
+          Doc, [revs, attachments, follows, att_encoding_info | Options])),
     {ContentType, Len} = couch_doc:len_doc_to_multi_part_stream(Boundary,
-        JsonBytes, Doc#doc.atts, false),
+        JsonBytes, Doc#doc.atts, true),
     Headers = case lists:member(delay_commit, Options) of
     true ->
         [{"X-Couch-Full-Commit", "false"}];
@@ -400,6 +401,8 @@ options_to_query_args([delay_commit | Re
     options_to_query_args(Rest, Acc);
 options_to_query_args([revs | Rest], Acc) ->
     options_to_query_args(Rest, [{"revs", "true"} | Acc]);
+options_to_query_args([att_encoding_info | Rest], Acc) ->
+    options_to_query_args(Rest, [{"att_encoding_info", "true"} | Acc]);
 options_to_query_args([{open_revs, all} | Rest], Acc) ->
     options_to_query_args(Rest, [{"open_revs", "all"} | Acc]);
 options_to_query_args([{open_revs, Revs} | Rest], Acc) ->
@@ -613,7 +616,7 @@ stream_doc({JsonBytes, Atts, Boundary, L
                 receive {get_data, From} ->
                     From ! {data, Data}
                 end
-            end, false),
+            end, true),
         unlink(Self)
     end),
     erlang:put({doc_streamer, Boundary}, DocStreamer),

Modified: couchdb/branches/new_replicator/src/couchdb/couch_httpd_db.erl
URL: http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_httpd_db.erl?rev=1035636&r1=1035635&r2=1035636&view=diff
==============================================================================
--- couchdb/branches/new_replicator/src/couchdb/couch_httpd_db.erl (original)
+++ couchdb/branches/new_replicator/src/couchdb/couch_httpd_db.erl Tue Nov 16 13:42:28 2010
@@ -736,14 +736,16 @@ send_doc_efficiently(Req, #doc{atts=Atts
             send_json(Req, 200, Headers, couch_doc:to_json_obj(Doc, Options));
         true ->
             Boundary = couch_uuids:random(),
+            CompressedAtts = lists:member(att_encoding_info, Options),
             JsonBytes = ?JSON_ENCODE(couch_doc:to_json_obj(Doc, 
                     [attachments, follows|Options])),
             {ContentType, Len} = couch_doc:len_doc_to_multi_part_stream(
-                    Boundary,JsonBytes, Atts,false),
+                    Boundary,JsonBytes, Atts, CompressedAtts),
             CType = {<<"Content-Type">>, ContentType},
             {ok, Resp} = start_response_length(Req, 200, [CType|Headers], Len),
-            couch_doc:doc_to_multi_part_stream(Boundary,JsonBytes,Atts,
-                    fun(Data) -> couch_httpd:send(Resp, Data) end, false)
+            couch_doc:doc_to_multi_part_stream(
+                Boundary, JsonBytes, Atts,
+                fun(Data) -> couch_httpd:send(Resp, Data) end, CompressedAtts)
         end;
     false ->
         send_json(Req, 200, Headers, couch_doc:to_json_obj(Doc, Options))
@@ -752,6 +754,7 @@ send_doc_efficiently(Req, #doc{atts=Atts
 send_docs_multipart(Req, Results, Options) ->
     OuterBoundary = couch_uuids:random(),
     InnerBoundary = couch_uuids:random(),
+    CompressedAtts = lists:member(att_encoding_info, Options),
     CType = {"Content-Type", 
         "multipart/mixed; boundary=\"" ++ ?b2l(OuterBoundary) ++ "\""},
     {ok, Resp} = start_chunked_response(Req, 200, [CType]),
@@ -761,12 +764,12 @@ send_docs_multipart(Req, Results, Option
             JsonBytes = ?JSON_ENCODE(couch_doc:to_json_obj(Doc, 
                     [attachments,follows|Options])),
             {ContentType, _Len} = couch_doc:len_doc_to_multi_part_stream(
-                    InnerBoundary, JsonBytes, Atts, false),
+                    InnerBoundary, JsonBytes, Atts, CompressedAtts),
             couch_httpd:send_chunk(Resp, <<"\r\nContent-Type: ",
                     ContentType/binary, "\r\n\r\n">>),
             couch_doc:doc_to_multi_part_stream(InnerBoundary, JsonBytes, Atts,
                     fun(Data) -> couch_httpd:send_chunk(Resp, Data)
-                    end, false),
+                    end, CompressedAtts),
              couch_httpd:send_chunk(Resp, <<"\r\n--", OuterBoundary/binary>>);
         ({{not_found, missing}, RevId}) ->
              RevStr = couch_doc:rev_to_str(RevId),