couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dav...@apache.org
Subject [1/2] fabric commit: updated refs/heads/master to 475b90c
Date Tue, 24 May 2016 16:57:04 GMT
Repository: couchdb-fabric
Updated Branches:
  refs/heads/master d20eafda9 -> 475b90ccd


Fix fabric_doc_open_revs

When a user specified multiple revisions on a single branch to
fabric_doc_open_revs it would throw a function clause exception in
lists:zipwith/3. This was due to a bad assumption that there would only
ever be exactly one revision for every input revision.

Due to the possibility of having zero or more revisions for a given
revision when using latest=true this code had to be changed fairly
significantly.

COUCHDB-2863


Project: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/commit/9a1d0c54
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/tree/9a1d0c54
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/diff/9a1d0c54

Branch: refs/heads/master
Commit: 9a1d0c54c78cef70d1a3992f3c71ef2878c498e4
Parents: a6d07a1
Author: Paul J. Davis <paul.joseph.davis@gmail.com>
Authored: Thu Apr 21 15:42:27 2016 -0500
Committer: Paul J. Davis <paul.joseph.davis@gmail.com>
Committed: Tue May 17 10:53:25 2016 -0500

----------------------------------------------------------------------
 src/fabric_doc_open_revs.erl | 582 +++++++++++++++++++++++---------------
 1 file changed, 355 insertions(+), 227 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/9a1d0c54/src/fabric_doc_open_revs.erl
----------------------------------------------------------------------
diff --git a/src/fabric_doc_open_revs.erl b/src/fabric_doc_open_revs.erl
index 4d493e4..4f0bf89 100644
--- a/src/fabric_doc_open_revs.erl
+++ b/src/fabric_doc_open_revs.erl
@@ -27,7 +27,8 @@
     r,
     revs,
     latest,
-    replies = []
+    replies = [],
+    repair = false
 }).
 
 go(DbName, Id, Revs, Options) ->
@@ -41,7 +42,7 @@ go(DbName, Id, Revs, Options) ->
         r = list_to_integer(R),
         revs = Revs,
         latest = lists:member(latest, Options),
-        replies = case Revs of all -> []; Revs -> [{Rev,[]} || Rev <- Revs] end
+        replies = []
     },
     RexiMon = fabric_util:create_monitors(Workers),
     try fabric_util:recv(Workers, #shard.ref, fun handle_message/3, State) of
@@ -56,266 +57,393 @@ go(DbName, Id, Revs, Options) ->
         rexi_monitor:stop(RexiMon)
     end.
 
+
 handle_message({rexi_DOWN, _, {_,NodeRef},_}, _Worker, #state{workers=Workers}=State) ->
-    NewWorkers = lists:keydelete(NodeRef, #shard.node, Workers),
-    skip(State#state{workers=NewWorkers});
+    NewState = State#state{
+        workers = lists:keydelete(NodeRef, #shard.node, Workers)
+    },
+    handle_message({ok, []}, nil, NewState);
+
 handle_message({rexi_EXIT, _}, Worker, #state{workers=Workers}=State) ->
-    skip(State#state{workers=lists:delete(Worker,Workers)});
-handle_message({ok, RawReplies}, Worker, #state{revs = all} = State) ->
+    NewState = State#state{
+        workers = lists:delete(Worker, Workers)
+    },
+    handle_message({ok, []}, nil, NewState);
+
+handle_message({ok, RawReplies}, Worker, State) ->
     #state{
         dbname = DbName,
         reply_count = ReplyCount,
         worker_count = WorkerCount,
         workers = Workers,
-        replies = All0,
-        r = R
+        replies = PrevReplies,
+        r = R,
+        revs = Revs,
+        latest = Latest,
+        repair = InRepair
     } = State,
-    All = lists:foldl(fun(Reply,D) -> fabric_util:update_counter(Reply,1,D) end,
-        All0, RawReplies),
-    Reduced = fabric_util:remove_ancestors(All, []),
-    Complete = (ReplyCount =:= (WorkerCount - 1)),
-    QuorumMet = lists:all(fun({_,{_, C}}) -> C >= R end, Reduced),
-    case Reduced of All when QuorumMet andalso ReplyCount =:= (R-1) ->
-        Repair = false;
-    _ ->
-        Repair = [D || {_,{{ok,D}, _}} <- Reduced]
+
+    IsTree = Revs == all orelse Latest,
+
+    {NewReplies, QuorumMet, Repair} = case IsTree of
+        true ->
+            {NewReplies0, AllInternal, Repair0} =
+                    tree_replies(PrevReplies, tree_sort(RawReplies)),
+            NumLeafs = couch_key_tree:count_leafs(PrevReplies),
+            SameNumRevs = length(RawReplies) == NumLeafs,
+            QMet = AllInternal andalso SameNumRevs andalso ReplyCount + 1 >= R,
+            {NewReplies0, QMet, Repair0};
+        false ->
+            {NewReplies0, MinCount} = dict_replies(PrevReplies, RawReplies),
+            {NewReplies0, MinCount >= R, false}
     end,
-    case maybe_reply(DbName, Reduced, Complete, Repair, R) of
-    noreply ->
-        {ok, State#state{replies = All, reply_count = ReplyCount+1,
-                        workers = lists:delete(Worker,Workers)}};
-    {reply, FinalReply} ->
-        fabric_util:cleanup(lists:delete(Worker,Workers)),
-        {stop, FinalReply}
-    end;
-handle_message({ok, RawReplies0}, Worker, State) ->
-    % we've got an explicit revision list, but if latest=true the workers may
-    % return a descendant of the requested revision.  Take advantage of the
-    % fact that revisions are returned in order to keep track.
-    RawReplies = strip_not_found_missing(RawReplies0),
-    #state{
-        dbname = DbName,
-        reply_count = ReplyCount,
-        worker_count = WorkerCount,
-        workers = Workers,
-        replies = All0,
-        r = R
-    } = State,
-    All = lists:zipwith(fun({Rev, D}, Reply) ->
-        if Reply =:= error -> {Rev, D}; true ->
-            {Rev, fabric_util:update_counter(Reply, 1, D)}
-        end
-    end, All0, RawReplies),
-    Reduced = [fabric_util:remove_ancestors(X, []) || {_, X} <- All],
-    FinalReplies = [choose_winner(X, R) || X <- Reduced, X =/= []],
+
     Complete = (ReplyCount =:= (WorkerCount - 1)),
-    case is_repair_needed(All, FinalReplies) of
-    true ->
-        Repair = [D || {_,{{ok,D}, _}} <- lists:flatten(Reduced)];
-    false ->
-        Repair = false
-    end,
-    case maybe_reply(DbName, FinalReplies, Complete, Repair, R) of
-    noreply ->
-        {ok, State#state{replies = All, reply_count = ReplyCount+1,
-                        workers=lists:delete(Worker,Workers)}};
-    {reply, FinalReply} ->
-        fabric_util:cleanup(lists:delete(Worker,Workers)),
-        {stop, FinalReply}
+
+    case QuorumMet orelse Complete of
+        true ->
+            fabric_util:cleanup(lists:delete(Worker, Workers)),
+            maybe_read_repair(
+                    DbName,
+                    IsTree,
+                    NewReplies,
+                    ReplyCount + 1,
+                    InRepair orelse Repair
+                ),
+            {stop, format_reply(IsTree, NewReplies)};
+        false ->
+            {ok, State#state{
+                replies = NewReplies,
+                reply_count = ReplyCount + 1,
+                workers = lists:delete(Worker, Workers),
+                repair = InRepair orelse Repair
+            }}
     end.
 
-skip(#state{revs=all} = State) ->
-    handle_message({ok, []}, nil, State);
-skip(#state{revs=Revs} = State) ->
-    handle_message({ok, [error || _Rev <- Revs]}, nil, State).
-
-maybe_reply(_, [], false, _, _) ->
-    noreply;
-maybe_reply(_, [], true, _, _) ->
-    {reply, {ok, []}};
-maybe_reply(DbName, ReplyDict, Complete, RepairDocs, R) ->
-    case Complete orelse lists:all(fun({_,{_, C}}) -> C >= R end, ReplyDict) of
-    true ->
-        maybe_execute_read_repair(DbName, RepairDocs),
-        {reply, unstrip_not_found_missing(extract_replies(ReplyDict))};
-    false ->
-        noreply
+
+tree_replies(RevTree, []) ->
+    {RevTree, true, false};
+
+tree_replies(RevTree0, [{ok, Doc} | Rest]) ->
+    {RevTree1, Done, Repair} = tree_replies(RevTree0, Rest),
+    Path = couch_doc:to_path(Doc),
+    case couch_key_tree:merge(RevTree1, Path) of
+        {RevTree2, internal_node} ->
+            {RevTree2, Done, Repair};
+        {RevTree2, new_leaf} ->
+            {RevTree2, Done, true};
+        {RevTree2, _} ->
+            {RevTree2, false, true}
+    end;
+
+tree_replies(RevTree0, [{{not_found, missing}, {Pos, Rev}} | Rest]) ->
+    {RevTree1, Done, Repair} = tree_replies(RevTree0, Rest),
+    Node = {Rev, ?REV_MISSING, []},
+    Path = {Pos, Node},
+    case couch_key_tree:merge(RevTree1, Path) of
+        {RevTree2, internal_node} ->
+            {RevTree2, Done, true};
+        {RevTree2, _} ->
+            {RevTree2, false, Repair}
     end.
 
-extract_replies(Replies) ->
-    lists:map(fun({_,{Reply,_}}) -> Reply end, Replies).
 
-choose_winner(Options, R) ->
-    case lists:dropwhile(fun({_,{_Reply, C}}) -> C < R end, Options) of
-    [] ->
-        case [Elem || {_,{{ok, #doc{}}, _}} = Elem <- Options] of
+tree_sort(Replies) ->
+    SortFun = fun(A, B) -> sort_key(A) =< sort_key(B) end,
+    lists:sort(SortFun, Replies).
+
+
+sort_key({ok, #doc{revs = {Pos, [Rev | _]}}}) ->
+    {Pos, Rev};
+sort_key({{not_found, _}, {Pos, Rev}}) ->
+    {Pos, Rev}.
+
+
+dict_replies(Dict, []) ->
+    Counts = [Count || {_Key, {_Reply, Count}} <- Dict],
+    {Dict, lists:min(Counts)};
+
+dict_replies(Dict, [Reply | Rest]) ->
+    NewDict = fabric_util:update_counter(Reply, 1, Dict),
+    dict_replies(NewDict, Rest).
+
+
+maybe_read_repair(Db, IsTree, Replies, ReplyCount, DoRepair) ->
+    Docs = case IsTree of
+        true -> tree_repair_docs(Replies, DoRepair);
+        false -> dict_repair_docs(Replies, ReplyCount)
+    end,
+    case Docs of
         [] ->
-            hd(Options);
-        Docs ->
-            lists:last(lists:sort(Docs))
-        end;
-    [QuorumMet | _] ->
-        QuorumMet
+            ok;
+        _ ->
+            erlang:spawn(fun() -> read_repair(Db, Docs) end)
+    end.
+
+
+tree_repair_docs(_Replies, false) ->
+    [];
+
+tree_repair_docs(Replies, true) ->
+    Leafs = couch_key_tree:get_all_leafs(Replies),
+    [Doc || {Doc, {_Pos, _}} <- Leafs, is_record(Doc, doc)].
+
+
+dict_repair_docs(Replies, ReplyCount) ->
+    NeedsRepair = lists:any(fun({_, {_, C}}) -> C < ReplyCount end, Replies),
+    if not NeedsRepair -> []; true ->
+        [Doc || {_, {{ok, Doc}, _}} <- Replies]
     end.
 
-% repair needed if any reply other than the winner has been received for a rev
-is_repair_needed([], []) ->
-    false;
-is_repair_needed([{_Rev, [Reply]} | Tail1], [Reply | Tail2]) ->
-    is_repair_needed(Tail1, Tail2);
-is_repair_needed(_, _) ->
-    true.
-
-maybe_execute_read_repair(_Db, false) ->
-    ok;
-maybe_execute_read_repair(Db, Docs) ->
-    [#doc{id=Id} | _] = Docs,
+
+read_repair(Db, Docs) ->
     Res = fabric:update_docs(Db, Docs, [replicated_changes, ?ADMIN_CTX]),
     case Res of
         {ok, []} ->
             couch_stats:increment_counter([fabric, read_repairs, success]);
         _ ->
             couch_stats:increment_counter([fabric, read_repairs, failure]),
+            [#doc{id = Id} | _] = Docs,
             couch_log:notice("read_repair ~s ~s ~p", [Db, Id, Res])
     end.
 
-% hackery required so that not_found sorts first
-strip_not_found_missing([]) ->
-    [];
-strip_not_found_missing([{{not_found, missing}, Rev} | Rest]) ->
-    [{not_found, Rev} | strip_not_found_missing(Rest)];
-strip_not_found_missing([Else | Rest]) ->
-    [Else | strip_not_found_missing(Rest)].
 
-unstrip_not_found_missing([]) ->
-    [];
-unstrip_not_found_missing([{not_found, Rev} | Rest]) ->
-    [{{not_found, missing}, Rev} | unstrip_not_found_missing(Rest)];
-unstrip_not_found_missing([Else | Rest]) ->
-    [Else | unstrip_not_found_missing(Rest)].
+format_reply(true, Replies) ->
+    tree_format_replies(Replies);
+
+format_reply(false, Replies) ->
+    dict_format_replies(Replies).
+
+
+tree_format_replies(RevTree) ->
+    Leafs = couch_key_tree:get_all_leafs(RevTree),
+    lists:sort(lists:map(fun(Reply) ->
+        case Reply of
+            {?REV_MISSING, {Pos, [Rev]}} ->
+                {{not_found, missing}, {Pos, Rev}};
+            {Doc, _} when is_record(Doc, doc) ->
+                {ok, Doc}
+        end
+    end, Leafs)).
+
+
+dict_format_replies(Dict) ->
+    lists:sort([Reply || {_, {Reply, _}} <- Dict]).
+
+
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
 
-all_revs_test() ->
+setup() ->
     config:start_link([]),
-    meck:new([fabric, couch_stats]),
+    meck:new([fabric, couch_stats, couch_log]),
     meck:expect(fabric, update_docs, fun(_, _, _) -> {ok, nil} end),
     meck:expect(couch_stats, increment_counter, fun(_) -> ok end),
-    meck:new(couch_log),
-    meck:expect(couch_log, notice, fun(_,_) -> ok end),
-
-    State0 = #state{worker_count = 3, workers=[nil,nil,nil], r = 2, revs = all},
-    Foo1 = {ok, #doc{revs = {1, [<<"foo">>]}}},
-    Foo2 = {ok, #doc{revs = {2, [<<"foo2">>, <<"foo">>]}}},
-    Bar1 = {ok, #doc{revs = {1, [<<"bar">>]}}},
-
-    % an empty worker response does not count as meeting quorum
-    ?assertMatch(
-        {ok, #state{workers=[nil,nil]}},
-        handle_message({ok, []}, nil, State0)
-    ),
-
-    ?assertMatch(
-        {ok, #state{workers=[nil, nil]}},
-        handle_message({ok, [Foo1, Bar1]}, nil, State0)
-    ),
-    {ok, State1} = handle_message({ok, [Foo1, Bar1]}, nil, State0),
-
-    % the normal case - workers agree
-    ?assertEqual(
-        {stop, [Bar1, Foo1]},
-        handle_message({ok, [Foo1, Bar1]}, nil, State1)
-    ),
-
-    % a case where the 2nd worker has a newer Foo - currently we're considering
-    % Foo to have reached quorum and execute_read_repair()
-    ?assertEqual(
-        {stop, [Bar1, Foo2]},
-        handle_message({ok, [Foo2, Bar1]}, nil, State1)
-    ),
-
-    % a case where quorum has not yet been reached for Foo
-    ?assertMatch(
-        {ok, #state{}},
-        handle_message({ok, [Bar1]}, nil, State1)
-    ),
-    {ok, State2} = handle_message({ok, [Bar1]}, nil, State1),
-
-    % still no quorum, but all workers have responded.  We include Foo1 in the
-    % response and execute_read_repair()
-    ?assertEqual(
-        {stop, [Bar1, Foo1]},
-        handle_message({ok, [Bar1]}, nil, State2)
-      ),
-    meck:unload([fabric, couch_log, couch_stats]),
+    meck:expect(couch_log, notice, fun(_, _) -> ok end).
+
+
+teardown(_) ->
+    (catch meck:unload([fabric, couch_stats, couch_log])),
     config:stop().
 
-specific_revs_test() ->
-    config:start_link([]),
-    meck:new([fabric, couch_stats]),
-    meck:expect(fabric, update_docs, fun(_, _, _) -> {ok, nil} end),
-    meck:expect(couch_stats, increment_counter, fun(_) -> ok end),
-    meck:new(couch_log),
-    meck:expect(couch_log, notice, fun(_,_) -> ok end),
 
-    Revs = [{1,<<"foo">>}, {1,<<"bar">>}, {1,<<"baz">>}],
-    State0 = #state{
+state0(Revs, Latest) ->
+    #state{
         worker_count = 3,
-        workers = [nil, nil, nil],
+        workers = [w1, w2, w3],
         r = 2,
         revs = Revs,
-        latest = false,
-        replies = [{Rev,[]} || Rev <- Revs]
-    },
-    Foo1 = {ok, #doc{revs = {1, [<<"foo">>]}}},
-    Foo2 = {ok, #doc{revs = {2, [<<"foo2">>, <<"foo">>]}}},
-    Bar1 = {ok, #doc{revs = {1, [<<"bar">>]}}},
-    Baz1 = {{not_found, missing}, {1,<<"baz">>}},
-    Baz2 = {ok, #doc{revs = {1, [<<"baz">>]}}},
-
-    ?assertMatch(
-        {ok, #state{}},
-        handle_message({ok, [Foo1, Bar1, Baz1]}, nil, State0)
-    ),
-    {ok, State1} = handle_message({ok, [Foo1, Bar1, Baz1]}, nil, State0),
-
-    % the normal case - workers agree
-    ?assertEqual(
-        {stop, [Foo1, Bar1, Baz1]},
-        handle_message({ok, [Foo1, Bar1, Baz1]}, nil, State1)
-    ),
-
-    % latest=true, worker responds with Foo2 and we return it
-    State0L = State0#state{latest = true},
-    ?assertMatch(
-        {ok, #state{}},
-        handle_message({ok, [Foo2, Bar1, Baz1]}, nil, State0L)
-    ),
-    {ok, State1L} = handle_message({ok, [Foo2, Bar1, Baz1]}, nil, State0L),
-    ?assertEqual(
-        {stop, [Foo2, Bar1, Baz1]},
-        handle_message({ok, [Foo2, Bar1, Baz1]}, nil, State1L)
-    ),
-
-    % Foo1 is included in the read quorum for Foo2
-    ?assertEqual(
-        {stop, [Foo2, Bar1, Baz1]},
-        handle_message({ok, [Foo1, Bar1, Baz1]}, nil, State1L)
-    ),
-
-    % {not_found, missing} is included in the quorum for any found revision
-    ?assertEqual(
-        {stop, [Foo2, Bar1, Baz2]},
-        handle_message({ok, [Foo2, Bar1, Baz2]}, nil, State1L)
-    ),
-
-    % a worker failure is skipped
-    ?assertMatch(
-        {ok, #state{}},
-        handle_message({rexi_EXIT, foo}, nil, State1L)
-    ),
-    {ok, State2L} = handle_message({rexi_EXIT, foo}, nil, State1L),
-    ?assertEqual(
-        {stop, [Foo2, Bar1, Baz2]},
-        handle_message({ok, [Foo2, Bar1, Baz2]}, nil, State2L)
-      ),
-    meck:unload([fabric, couch_log, couch_stats]),
-    config:stop().
+        latest = Latest
+    }.
+
+
+revs() -> [{1,<<"foo">>}, {1,<<"bar">>}, {1,<<"baz">>}].
+
+
+foo1() -> {ok, #doc{revs = {1, [<<"foo">>]}}}.
+foo2() -> {ok, #doc{revs = {2, [<<"foo2">>, <<"foo">>]}}}.
+bar1() -> {ok, #doc{revs = {1, [<<"bar">>]}}}.
+bazNF() -> {{not_found, missing}, {1,<<"baz">>}}.
+baz1() -> {ok, #doc{revs = {1, [<<"baz">>]}}}.
+
+
+
+open_doc_revs_test_() ->
+    {
+        foreach,
+        fun setup/0,
+        fun teardown/1,
+        [
+            check_empty_response_not_quorum(),
+            check_basic_response(),
+            check_finish_quorum(),
+            check_finish_quorum_newer(),
+            check_no_quorum_on_second(),
+            check_done_on_third(),
+            check_specific_revs_first_msg(),
+            check_revs_done_on_agreement(),
+            check_latest_true(),
+            check_ancestor_counted_in_quorum(),
+            check_not_found_counts_for_descendant(),
+            check_worker_error_skipped()
+        ]
+    }.
+
+
+% Tests for revs=all
+
+
+check_empty_response_not_quorum() ->
+    % Simple smoke test that we don't think we're
+    % done with a first empty response
+    ?_assertMatch(
+        {ok, #state{workers = [w2, w3]}},
+        handle_message({ok, []}, w1, state0(all, false))
+    ).
+
+
+check_basic_response() ->
+    % Check that we've handle a response
+    ?_assertMatch(
+        {ok, #state{reply_count = 1, workers = [w2, w3]}},
+        handle_message({ok, [foo1(), bar1()]}, w1, state0(all, false))
+    ).
+
+
+check_finish_quorum() ->
+    % Two messages with the same revisions means we're done
+    ?_test(begin
+        S0 = state0(all, false),
+        {ok, S1} = handle_message({ok, [foo1(), bar1()]}, w1, S0),
+        Expect = {stop, [bar1(), foo1()]},
+        ?assertEqual(Expect, handle_message({ok, [foo1(), bar1()]}, w2, S1))
+    end).
+
+
+check_finish_quorum_newer() ->
+    % We count a descendant of a revision for quorum so
+    % foo1 should count for foo2 which means we're finished.
+    % We also validate that read_repair was triggered.
+    ?_test(begin
+        S0 = state0(all, false),
+        {ok, S1} = handle_message({ok, [foo1(), bar1()]}, w1, S0),
+        Expect = {stop, [bar1(), foo2()]},
+        ?assertEqual(Expect, handle_message({ok, [foo2(), bar1()]}, w2, S1)),
+        ?assertMatch(
+            [{_, {fabric, update_docs, [_, _, _]}, _}],
+            meck:history(fabric)
+        )
+    end).
+
+
+check_no_quorum_on_second() ->
+    % Quorum not yet met for the foo revision so we
+    % would wait for w3
+    ?_test(begin
+        S0 = state0(all, false),
+        {ok, S1} = handle_message({ok, [foo1(), bar1()]}, w1, S0),
+        ?assertMatch(
+            {ok, #state{workers = [w3]}},
+            handle_message({ok, [bar1()]}, w2, S1)
+        )
+    end).
+
+
+check_done_on_third() ->
+    % The third message of three means we're done no matter
+    % what. Every revision seen in this pattern should be
+    % included.
+    ?_test(begin
+        S0 = state0(all, false),
+        {ok, S1} = handle_message({ok, [foo1(), bar1()]}, w1, S0),
+        {ok, S2} = handle_message({ok, [bar1()]}, w2, S1),
+        Expect = {stop, [bar1(), foo1()]},
+        ?assertEqual(Expect, handle_message({ok, [bar1()]}, w3, S2))
+    end).
+
+
+% Tests for a specific list of revs
+
+
+check_specific_revs_first_msg() ->
+    ?_test(begin
+        S0 = state0(revs(), false),
+        ?assertMatch(
+            {ok, #state{reply_count = 1, workers = [w2, w3]}},
+            handle_message({ok, [foo1(), bar1(), bazNF()]}, w1, S0)
+        )
+    end).
+
+
+check_revs_done_on_agreement() ->
+    ?_test(begin
+        S0 = state0(revs(), false),
+        Msg = {ok, [foo1(), bar1(), bazNF()]},
+        {ok, S1} = handle_message(Msg, w1, S0),
+        Expect = {stop, [bar1(), foo1(), bazNF()]},
+        ?assertEqual(Expect, handle_message(Msg, w2, S1))
+    end).
+
+
+check_latest_true() ->
+    ?_test(begin
+        S0 = state0(revs(), true),
+        Msg1 = {ok, [foo2(), bar1(), bazNF()]},
+        Msg2 = {ok, [foo2(), bar1(), bazNF()]},
+        {ok, S1} = handle_message(Msg1, w1, S0),
+        Expect = {stop, [bar1(), foo2(), bazNF()]},
+        ?assertEqual(Expect, handle_message(Msg2, w2, S1))
+    end).
+
+
+check_ancestor_counted_in_quorum() ->
+    ?_test(begin
+        S0 = state0(revs(), true),
+        Msg1 = {ok, [foo1(), bar1(), bazNF()]},
+        Msg2 = {ok, [foo2(), bar1(), bazNF()]},
+        Expect = {stop, [bar1(), foo2(), bazNF()]},
+
+        % Older first
+        {ok, S1} = handle_message(Msg1, w1, S0),
+        ?assertEqual(Expect, handle_message(Msg2, w2, S1)),
+
+        % Newer first
+        {ok, S2} = handle_message(Msg2, w2, S0),
+        ?assertEqual(Expect, handle_message(Msg1, w1, S2))
+    end).
+
+
+check_not_found_counts_for_descendant() ->
+    ?_test(begin
+        S0 = state0(revs(), true),
+        Msg1 = {ok, [foo1(), bar1(), bazNF()]},
+        Msg2 = {ok, [foo1(), bar1(), baz1()]},
+        Expect = {stop, [bar1(), baz1(), foo1()]},
+
+        % not_found first
+        {ok, S1} = handle_message(Msg1, w1, S0),
+        ?assertEqual(Expect, handle_message(Msg2, w2, S1)),
+
+        % not_found second
+        {ok, S2} = handle_message(Msg2, w2, S0),
+        ?assertEqual(Expect, handle_message(Msg1, w1, S2))
+    end).
+
+
+check_worker_error_skipped() ->
+    ?_test(begin
+        S0 = state0(revs(), true),
+        Msg1 = {ok, [foo1(), bar1(), baz1()]},
+        Msg2 = {rexi_EXIT, reason},
+        Msg3 = {ok, [foo1(), bar1(), baz1()]},
+        Expect = {stop, [bar1(), baz1(), foo1()]},
+
+        {ok, S1} = handle_message(Msg1, w1, S0),
+        {ok, S2} = handle_message(Msg2, w2, S1),
+        ?assertEqual(Expect, handle_message(Msg3, w2, S2))
+    end).
+
+
+-endif.


Mime
View raw message