couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bbast...@apache.org
Subject [2/5] couch-replicator commit: updated refs/heads/master to c9184cf
Date Fri, 31 Oct 2014 19:53:11 GMT
don't replicate removed document in a view changes

Conflicts:
	src/couch_replicator.erl
	src/couch_replicator_api_wrap.erl


Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/7390c7ec
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/7390c7ec
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/7390c7ec

Branch: refs/heads/master
Commit: 7390c7eccd19552ab842bc6b0a3c31fbbfc724c4
Parents: 3e3e2cb
Author: benoitc <bchesneau@gmail.com>
Authored: Thu Jul 3 13:08:34 2014 +0200
Committer: Benjamin Bastian <benjamin.bastian@gmail.com>
Committed: Thu Oct 30 13:29:56 2014 -0700

----------------------------------------------------------------------
 src/couch_replicator.erl          | 53 ++++++++++++++++++++++++++++++++++
 src/couch_replicator_api_wrap.erl | 12 ++++++--
 src/couch_replicator_worker.erl   | 15 ++++++----
 3 files changed, 72 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/7390c7ec/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl
index f7e9d27..6992fff 100644
--- a/src/couch_replicator.erl
+++ b/src/couch_replicator.erl
@@ -647,6 +647,59 @@ fold_replication_logs([Db | Rest] = Dbs, Vsn, LogId, NewId, Rep, Acc)
->
     end.
 
 
+spawn_changes_reader(StartSeq, #httpdb{} = Db, ChangesQueue, Options) ->
+    spawn_link(fun() ->
+        put(last_seq, StartSeq),
+        put(retries_left, Db#httpdb.retries),
+        read_changes(StartSeq, Db#httpdb{retries = 0}, ChangesQueue, Options)
+    end);
+spawn_changes_reader(StartSeq, Db, ChangesQueue, Options) ->
+    spawn_link(fun() ->
+        read_changes(StartSeq, Db, ChangesQueue, Options)
+    end).
+
+read_changes(StartSeq, Db, ChangesQueue, Options) ->
+    try
+        couch_replicator_api_wrap:changes_since(Db, all_docs, StartSeq, fun
+            (#doc_info{high_seq = Seq, revs = []}) ->
+                put(last_seq, Seq);
+            (#doc_info{high_seq = Seq, id = Id} = DocInfo) ->
+                case Id of
+                <<>> ->
+                    % Previous CouchDB releases had a bug which allowed a doc
+                    % with an empty ID to be inserted into databases. Such doc
+                    % is impossible to GET.
+                    ?LOG_ERROR("Replicator: ignoring document with empty ID in "
+                        "source database `~s` (_changes sequence ~p)",
+                        [couch_replicator_api_wrap:db_uri(Db), Seq]);
+                _ ->
+                    ok = couch_work_queue:queue(ChangesQueue, DocInfo)
+                end,
+                put(last_seq, Seq)
+            end, Options),
+        couch_work_queue:close(ChangesQueue)
+    catch exit:{http_request_failed, _, _, _} = Error ->
+        case get(retries_left) of
+        N when N > 0 ->
+            put(retries_left, N - 1),
+            LastSeq = get(last_seq),
+            Db2 = case LastSeq of
+            StartSeq ->
+                ?LOG_INFO("Retrying _changes request to source database ~s"
+                    " with since=~p in ~p seconds",
+                    [couch_replicator_api_wrap:db_uri(Db), LastSeq, Db#httpdb.wait / 1000]),
+                ok = timer:sleep(Db#httpdb.wait),
+                Db#httpdb{wait = 2 * Db#httpdb.wait};
+            _ ->
+                ?LOG_INFO("Retrying _changes request to source database ~s"
+                    " with since=~p", [couch_replicator_api_wrap:db_uri(Db), LastSeq]),
+                Db
+            end,
+            read_changes(LastSeq, Db2, ChangesQueue, Options);
+        _ ->
+            exit(Error)
+        end
+    end.
 
 
 spawn_changes_manager(Parent, ChangesQueue, BatchSize) ->

http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/7390c7ec/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index 97e9b25..c35659d 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -839,12 +839,21 @@ json_to_doc_info({Props}) ->
     undefined ->
         {last_seq, get_value(<<"last_seq">>, Props)};
     Changes ->
-        RevsInfo = lists:map(
+        RevsInfo0 = lists:map(
             fun({Change}) ->
                 Rev = couch_doc:parse_rev(get_value(<<"rev">>, Change)),
                 Del = couch_replicator_utils:is_deleted(Change),
                 #rev_info{rev=Rev, deleted=Del}
             end, Changes),
+
+        RevsInfo = case get_value(<<"removed">>, Props) of
+            true ->
+                [_ | RevsInfo1] = RevsInfo0,
+                RevsInfo1;
+            _ ->
+                RevsInfo0
+        end,
+
         #doc_info{
             id = get_value(<<"id">>, Props),
             high_seq = get_value(<<"seq">>, Props),
@@ -852,7 +861,6 @@ json_to_doc_info({Props}) ->
         }
     end.
 
-
 bulk_results_to_errors(Docs, {ok, Results}, interactive_edit) ->
     lists:reverse(lists:foldl(
         fun({_, {ok, _}}, Acc) ->

http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/7390c7ec/src/couch_replicator_worker.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_worker.erl b/src/couch_replicator_worker.erl
index 7a6cc96..155e11d 100644
--- a/src/couch_replicator_worker.erl
+++ b/src/couch_replicator_worker.erl
@@ -491,12 +491,15 @@ flush_doc(Target, #doc{id = Id, revs = {Pos, [RevId | _]}} = Doc) ->
 
 
 find_missing(DocInfos, Target) ->
-    {IdRevs, AllRevsCount} = lists:foldr(
-        fun(#doc_info{id = Id, revs = RevsInfo}, {IdRevAcc, CountAcc}) ->
-            Revs = [Rev || #rev_info{rev = Rev} <- RevsInfo],
-            {[{Id, Revs} | IdRevAcc], CountAcc + length(Revs)}
-        end,
-        {[], 0}, DocInfos),
+    {IdRevs, AllRevsCount} = lists:foldr(fun
+                (#doc_info{revs = []}, {IdRevAcc, CountAcc}) ->
+                    {IdRevAcc, CountAcc};
+                (#doc_info{id = Id, revs = RevsInfo}, {IdRevAcc, CountAcc}) ->
+                    Revs = [Rev || #rev_info{rev = Rev} <- RevsInfo],
+                    {[{Id, Revs} | IdRevAcc], CountAcc + length(Revs)}
+            end, {[], 0}, DocInfos),
+
+
     {ok, Missing} = couch_replicator_api_wrap:get_missing_revs(Target, IdRevs),
     MissingRevsCount = lists:foldl(
         fun({_Id, MissingRevs, _PAs}, Acc) -> Acc + length(MissingRevs) end,


Mime
View raw message