couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From fdman...@apache.org
Subject svn commit: r1085178 - in /couchdb/trunk: THANKS src/couchdb/couch_api_wrap.erl src/couchdb/couch_replicator_doc_copier.erl
Date Thu, 24 Mar 2011 22:56:48 GMT
Author: fdmanana
Date: Thu Mar 24 22:56:47 2011
New Revision: 1085178

URL: http://svn.apache.org/viewvc?rev=1085178&view=rev
Log:
Replicator: deal gracefully with missing_stub errors

This issue was reported by Wayne Conrad at the users mailing list.
This patch was also tested by him and confirmed to fix the issue.

Thanks Wayne.


Modified:
    couchdb/trunk/THANKS
    couchdb/trunk/src/couchdb/couch_api_wrap.erl
    couchdb/trunk/src/couchdb/couch_replicator_doc_copier.erl

Modified: couchdb/trunk/THANKS
URL: http://svn.apache.org/viewvc/couchdb/trunk/THANKS?rev=1085178&r1=1085177&r2=1085178&view=diff
==============================================================================
--- couchdb/trunk/THANKS (original)
+++ couchdb/trunk/THANKS Thu Mar 24 22:56:47 2011
@@ -75,5 +75,6 @@ suggesting improvements or submitting ch
  * Benjamin Young <byoung@bigbluehat.com>
  * Gabriel Farrell <gsf747@gmail.com>
  * Mike Leddy <mike@loop.com.br>
+ * Wayne Conrad <wayne@databill.com>
 
 For a list of authors see the `AUTHORS` file.

Modified: couchdb/trunk/src/couchdb/couch_api_wrap.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_api_wrap.erl?rev=1085178&r1=1085177&r2=1085178&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_api_wrap.erl (original)
+++ couchdb/trunk/src/couchdb/couch_api_wrap.erl Thu Mar 24 22:56:47 2011
@@ -232,16 +232,18 @@ update_doc(#httpdb{} = HttpDb, #doc{id =
                 {ok, couch_doc:parse_rev(get_value(<<"rev">>, Props))};
             (409, _, _) ->
                 throw(conflict);
-            (_, _, {Props}) ->
-                {error, get_value(<<"error">>, Props)}
+            (Code, _, {Props}) ->
+                case {Code, get_value(<<"error">>, Props)} of
+                {401, <<"unauthorized">>} ->
+                    throw({unauthorized, get_value(<<"reason">>, Props)});
+                {412, <<"missing_stub">>} ->
+                    throw({missing_stub, get_value(<<"reason">>, Props)});
+                {_, Error} ->
+                    {error, Error}
+                end
         end);
 update_doc(Db, Doc, Options, Type) ->
-    try
-        couch_db:update_doc(Db, Doc, Options, Type)
-    catch
-    throw:{unauthorized, _} ->
-        {error, <<"unauthorized">>}
-    end.
+    couch_db:update_doc(Db, Doc, Options, Type).
 
 
 update_docs(Db, DocList, Options) ->

Modified: couchdb/trunk/src/couchdb/couch_replicator_doc_copier.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_replicator_doc_copier.erl?rev=1085178&r1=1085177&r2=1085178&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_replicator_doc_copier.erl (original)
+++ couchdb/trunk/src/couchdb/couch_replicator_doc_copier.erl Thu Mar 24 22:56:47 2011
@@ -279,19 +279,21 @@ local_process_batch([], _Source, Target,
 
 local_process_batch([{Seq, {Id, Revs, NotMissingCount, PAs}} | Rest],
     Source, Target, Batch, Stats, HighestSeqSeen) ->
-    {ok, DocList} = fetch_doc(
-        Source, {Id, Revs, PAs, Seq}, fun local_doc_handler/2, []),
+    {ok, {_, DocList, Written0, WriteFailures0}} = fetch_doc(
+        Source, {Id, Revs, PAs, Seq}, fun local_doc_handler/2,
+        {Target, [], 0, 0}),
+    Read = length(DocList) + Written0 + WriteFailures0,
     {Batch2, Written, WriteFailures} = lists:foldl(
         fun(Doc, {Batch0, W0, F0}) ->
             {Batch1, W, F} = maybe_flush_docs(Target, Batch0, Doc),
             {Batch1, W0 + W, F0 + F}
         end,
-        {Batch, 0, 0}, DocList),
+        {Batch, Written0, WriteFailures0}, DocList),
     Stats2 = Stats#rep_stats{
         missing_checked = Stats#rep_stats.missing_checked + length(Revs)
             + NotMissingCount,
         missing_found = Stats#rep_stats.missing_found + length(Revs),
-        docs_read = Stats#rep_stats.docs_read + length(DocList),
+        docs_read = Stats#rep_stats.docs_read + Read,
         docs_written = Stats#rep_stats.docs_written + Written,
         doc_write_failures = Stats#rep_stats.doc_write_failures + WriteFailures
     },
@@ -333,14 +335,33 @@ spawn_doc_reader(Source, Target, FetchPa
 
 
 fetch_doc(Source, {Id, Revs, PAs, _Seq}, DocHandler, Acc) ->
-    couch_api_wrap:open_doc_revs(
-        Source, Id, Revs, [{atts_since, PAs}], DocHandler, Acc).
+    try
+        couch_api_wrap:open_doc_revs(
+            Source, Id, Revs, [{atts_since, PAs}], DocHandler, Acc)
+    catch
+    throw:{missing_stub, _} ->
+        ?LOG_ERROR("Retrying fetch and update of document `~p` due to out of "
+            "sync attachment stubs. Missing revisions are: ~s",
+            [Id, couch_doc:revs_to_strs(Revs)]),
+        couch_api_wrap:open_doc_revs(Source, Id, Revs, [], DocHandler, Acc)
+    end.
 
 
-local_doc_handler({ok, Doc}, DocList) ->
-    [Doc | DocList];
-local_doc_handler(_, DocList) ->
-    DocList.
+local_doc_handler({ok, #doc{atts = []} = Doc}, {Target, DocList, W, F}) ->
+    {Target, [Doc | DocList], W, F};
+local_doc_handler({ok, Doc}, {Target, DocList, W, F}) ->
+    ?LOG_DEBUG("Worker flushing doc with attachments", []),
+    Target2 = open_db(Target),
+    Success = (flush_doc(Target2, Doc) =:= ok),
+    close_db(Target2),
+    case Success of
+    true ->
+        {Target, DocList, W + 1, F};
+    false ->
+        {Target, DocList, W, F + 1}
+    end;
+local_doc_handler(_, Acc) ->
+    Acc.
 
 
 remote_doc_handler({ok, #doc{atts = []} = Doc}, {Parent, _} = Acc) ->
@@ -351,6 +372,7 @@ remote_doc_handler({ok, Doc}, {Parent, T
     % source. The data property of each attachment is a function that starts
     % streaming the attachment data from the remote source, therefore it's
     % convenient to call it ASAP to avoid ibrowse inactivity timeouts.
+    ?LOG_DEBUG("Worker flushing doc with attachments", []),
     Target2 = open_db(Target),
     Success = (flush_doc(Target2, Doc) =:= ok),
     ok = gen_server:call(Parent, {doc_flushed, Success}, infinity),
@@ -418,7 +440,7 @@ maybe_flush_docs(#httpdb{} = Target,
         case flush_doc(Target, Doc) of
         ok ->
             {Batch, 1, 0};
-        error ->
+        _ ->
             {Batch, 0, 1}
         end;
     false ->
@@ -442,15 +464,6 @@ maybe_flush_docs(#db{} = Target, #batch{
         {#batch{}, Written, Failed};
     SizeAcc2 ->
         {#batch{docs = [Doc | DocAcc], size = SizeAcc2}, 0, 0}
-    end;
-
-maybe_flush_docs(#db{} = Target, Batch, Doc) ->
-    ?LOG_DEBUG("Worker flushing doc with attachments", []),
-    case flush_doc(Target, Doc) of
-    ok ->
-        {Batch, 1, 0};
-    error ->
-        {Batch, 0, 1}
     end.
 
 
@@ -470,14 +483,17 @@ flush_docs(Target, DocList) ->
         end, Errors),
     {length(DocList) - length(Errors), length(Errors)}.
 
-flush_doc(Target, Doc) ->
-    case couch_api_wrap:update_doc(Target, Doc, [], replicated_changes) of
+flush_doc(Target, #doc{id = Id} = Doc) ->
+    try couch_api_wrap:update_doc(Target, Doc, [], replicated_changes) of
     {ok, _} ->
         ok;
-    {error, <<"unauthorized">>} ->
+    Error ->
+        ?LOG_ERROR("Replicator: error writing document `~s` to `~s`: ~s",
+            [Id, couch_api_wrap:db_uri(Target), couch_util:to_binary(Error)]),
+        Error
+    catch
+    throw:{unauthorized, _} ->
         ?LOG_ERROR("Replicator: unauthorized to write document `~s` to `~s`",
-            [Doc#doc.id, couch_api_wrap:db_uri(Target)]),
-        error;
-    _ ->
-        error
+            [Id, couch_api_wrap:db_uri(Target)]),
+        {error, unauthorized}
     end.



Mime
View raw message