couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kocol...@apache.org
Subject svn commit: r802974 - in /couchdb/trunk/src/couchdb: couch_rep_httpc.erl couch_rep_writer.erl
Date Tue, 11 Aug 2009 01:51:07 GMT
Author: kocolosk
Date: Tue Aug 11 01:51:06 2009
New Revision: 802974

URL: http://svn.apache.org/viewvc?rev=802974&view=rev
Log:
stream _bulk_docs JSON body during replication

Modified:
    couchdb/trunk/src/couchdb/couch_rep_httpc.erl
    couchdb/trunk/src/couchdb/couch_rep_writer.erl

Modified: couchdb/trunk/src/couchdb/couch_rep_httpc.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_rep_httpc.erl?rev=802974&r1=802973&r2=802974&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_rep_httpc.erl (original)
+++ couchdb/trunk/src/couchdb/couch_rep_httpc.erl Tue Aug 11 01:51:06 2009
@@ -39,7 +39,14 @@
     {OAuthProps} ->
         [oauth_header(Url, Method, OAuthProps) | Headers0]
     end,
-    Body = if B =:= nil -> []; true -> iolist_to_binary(?JSON_ENCODE(B)) end,
+    Body = case B of
+    {Fun, InitialState} when is_function(Fun) ->
+        {Fun, InitialState};
+    nil ->
+        [];
+    _Else ->
+        iolist_to_binary(?JSON_ENCODE(B)) 
+    end,
     Resp = case Conn of
     nil ->
         ibrowse:send_req(Url, Headers, Method, Body, Opts, infinity);

Modified: couchdb/trunk/src/couchdb/couch_rep_writer.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_rep_writer.erl?rev=802974&r1=802973&r2=802974&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_rep_writer.erl (original)
+++ couchdb/trunk/src/couchdb/couch_rep_writer.erl Tue Aug 11 01:51:06 2009
@@ -19,15 +19,30 @@
 start_link(Parent, Target, Reader, _PostProps) ->
     {ok, spawn_link(fun() -> writer_loop(Parent, Reader, Target) end)}.
 
+make_chunk(Data) when is_list(Data) ->
+    make_chunk(list_to_binary(Data));
+make_chunk(Data) ->
+    [ibrowse_lib:dec2hex(4, size(Data)), "\r\n", Data, "\r\n"].
+
+upload_docs({start, Docs}) ->
+    Data = make_chunk(<<"{\"new_edits\":false, \"docs\":[">>),
+    {ok, Data, {continue, Docs, ""}};
+upload_docs({continue, [Doc|Rest], Prepend}) ->
+    JsonDoc = ?JSON_ENCODE(couch_doc:to_json_obj(Doc, [revs,attachments])),
+    {ok, make_chunk([Prepend, JsonDoc]), {continue, Rest, ","}};
+upload_docs({continue, [], _}) ->
+    {ok, make_chunk(<<"]}\n">>), last_chunk};
+upload_docs(last_chunk) ->
+    {ok, "0\r\n\r\n", finish};
+upload_docs(finish) ->
+    eof.
+    
 writer_loop(Parent, Reader, Target) ->
-    % ?LOG_DEBUG("writer loop begin", []),
     case couch_rep_reader:next(Reader) of
     {complete, FinalSeq} ->
-        % ?LOG_INFO("writer terminating normally", []),
         Parent ! {writer_checkpoint, FinalSeq},
         ok;
     {HighSeq, Docs} ->
-        % ?LOG_DEBUG("writer loop trying to write ~p", [Docs]),
         DocCount = length(Docs),
         try write_docs(Target, Docs) of
         {ok, []} ->
@@ -47,11 +62,11 @@
     end.
 
 write_docs(#http_db{} = Db, Docs) ->
-    JsonDocs = [couch_doc:to_json_obj(Doc, [revs,attachments]) || Doc <- Docs],
     ErrorsJson = couch_rep_httpc:request(Db#http_db{
         resource = "_bulk_docs",
         method = post,
-        body = {[{new_edits, false}, {docs, JsonDocs}]}
+        body = {fun upload_docs/1, {start, Docs}},
+        headers = [{"transfer-encoding", "chunked"} | Db#http_db.headers]
     }),
     ErrorsList =
     lists:map(



Mime
View raw message