couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rnew...@apache.org
Subject [01/14] Move files out of test/couchdb into top level test/ folder
Date Fri, 05 Sep 2014 21:10:48 GMT
Repository: couchdb-couch-replicator
Updated Branches:
  refs/heads/master aafb5f9d9 -> 3cf0b131b


http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2850b946/test/couch_replicator_large_atts_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_large_atts_tests.erl b/test/couch_replicator_large_atts_tests.erl
new file mode 100644
index 0000000..ed7ec50
--- /dev/null
+++ b/test/couch_replicator_large_atts_tests.erl
@@ -0,0 +1,218 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_large_atts_tests).
+
+-include("../../../test/couchdb/couch_eunit.hrl").
+-include_lib("couchdb/couch_db.hrl").
+
+-define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
+-define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
+-define(ATT_SIZE_1, 2 * 1024 * 1024).
+-define(ATT_SIZE_2, round(6.6 * 1024 * 1024)).
+-define(DOCS_COUNT, 11).
+-define(TIMEOUT_EUNIT, 30).
+-define(TIMEOUT_STOP, 1000).
+
+
+setup() ->
+    DbName = ?tempdb(),
+    {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
+    ok = couch_db:close(Db),
+    DbName.
+
+setup(local) ->
+    setup();
+setup(remote) ->
+    {remote, setup()};
+setup({A, B}) ->
+    {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
+    couch_config:set("attachments", "compressible_types", "text/*", false),
+    Source = setup(A),
+    Target = setup(B),
+    {Source, Target}.
+
+teardown({remote, DbName}) ->
+    teardown(DbName);
+teardown(DbName) ->
+    ok = couch_server:delete(DbName, [?ADMIN_USER]),
+    ok.
+
+teardown(_, {Source, Target}) ->
+    teardown(Source),
+    teardown(Target),
+
+    Pid = whereis(couch_server_sup),
+    erlang:monitor(process, Pid),
+    couch_server_sup:stop(),
+    receive
+        {'DOWN', _, _, Pid, _} ->
+            ok
+    after ?TIMEOUT_STOP ->
+        throw({timeout, server_stop})
+    end.
+
+
+large_atts_test_() ->
+    Pairs = [{local, local}, {local, remote},
+             {remote, local}, {remote, remote}],
+    {
+        "Replicate docs with large attachments",
+        {
+            foreachx,
+            fun setup/1, fun teardown/2,
+            [{Pair, fun should_populate_replicate_compact/2}
+             || Pair <- Pairs]
+        }
+    }.
+
+
+should_populate_replicate_compact({From, To}, {Source, Target}) ->
+    {lists:flatten(io_lib:format("~p -> ~p", [From, To])),
+     {inorder, [should_populate_source(Source),
+                should_replicate(Source, Target),
+                should_compare_databases(Source, Target)]}}.
+
+should_populate_source({remote, Source}) ->
+    should_populate_source(Source);
+should_populate_source(Source) ->
+    {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source, ?DOCS_COUNT))}.
+
+should_replicate({remote, Source}, Target) ->
+    should_replicate(db_url(Source), Target);
+should_replicate(Source, {remote, Target}) ->
+    should_replicate(Source, db_url(Target));
+should_replicate(Source, Target) ->
+    {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
+
+should_compare_databases({remote, Source}, Target) ->
+    should_compare_databases(Source, Target);
+should_compare_databases(Source, {remote, Target}) ->
+    should_compare_databases(Source, Target);
+should_compare_databases(Source, Target) ->
+    {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}.
+
+
+populate_db(DbName, DocCount) ->
+    {ok, Db} = couch_db:open_int(DbName, []),
+    Docs = lists:foldl(
+        fun(DocIdCounter, Acc) ->
+            Doc = #doc{
+                id = iolist_to_binary(["doc", integer_to_list(DocIdCounter)]),
+                body = {[]},
+                atts = [
+                    att(<<"att1">>, ?ATT_SIZE_1, <<"text/plain">>),
+                    att(<<"att2">>, ?ATT_SIZE_2, <<"app/binary">>)
+                ]
+            },
+            [Doc | Acc]
+        end,
+        [], lists:seq(1, DocCount)),
+    {ok, _} = couch_db:update_docs(Db, Docs, []),
+    couch_db:close(Db).
+
+compare_dbs(Source, Target) ->
+    {ok, SourceDb} = couch_db:open_int(Source, []),
+    {ok, TargetDb} = couch_db:open_int(Target, []),
+
+    Fun = fun(FullDocInfo, _, Acc) ->
+        {ok, DocSource} = couch_db:open_doc(SourceDb, FullDocInfo),
+        Id = DocSource#doc.id,
+
+        {ok, DocTarget} = couch_db:open_doc(TargetDb, Id),
+        ?assertEqual(DocSource#doc.body, DocTarget#doc.body),
+
+        #doc{atts = SourceAtts} = DocSource,
+        #doc{atts = TargetAtts} = DocTarget,
+        ?assertEqual(lists:sort([N || #att{name = N} <- SourceAtts]),
+                     lists:sort([N || #att{name = N} <- TargetAtts])),
+
+        FunCompareAtts = fun(#att{name = AttName} = Att) ->
+            {ok, AttTarget} = find_att(TargetAtts, AttName),
+            SourceMd5 = att_md5(Att),
+            TargetMd5 = att_md5(AttTarget),
+            case AttName of
+                <<"att1">> ->
+                    ?assertEqual(gzip, Att#att.encoding),
+                    ?assertEqual(gzip, AttTarget#att.encoding),
+                    DecSourceMd5 = att_decoded_md5(Att),
+                    DecTargetMd5 = att_decoded_md5(AttTarget),
+                    ?assertEqual(DecSourceMd5, DecTargetMd5);
+                _ ->
+                    ?assertEqual(identity, Att#att.encoding),
+                    ?assertEqual(identity, AttTarget#att.encoding)
+            end,
+            ?assertEqual(SourceMd5, TargetMd5),
+            ?assert(is_integer(Att#att.disk_len)),
+            ?assert(is_integer(Att#att.att_len)),
+            ?assert(is_integer(AttTarget#att.disk_len)),
+            ?assert(is_integer(AttTarget#att.att_len)),
+            ?assertEqual(Att#att.disk_len, AttTarget#att.disk_len),
+            ?assertEqual(Att#att.att_len, AttTarget#att.att_len),
+            ?assertEqual(Att#att.type, AttTarget#att.type),
+            ?assertEqual(Att#att.md5, AttTarget#att.md5)
+        end,
+
+        lists:foreach(FunCompareAtts, SourceAtts),
+
+        {ok, Acc}
+    end,
+
+    {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
+    ok = couch_db:close(SourceDb),
+    ok = couch_db:close(TargetDb).
+
+att(Name, Size, Type) ->
+    #att{
+        name = Name,
+        type = Type,
+        att_len = Size,
+        data = fun(Count) -> crypto:rand_bytes(Count) end
+    }.
+
+find_att([], _Name) ->
+    nil;
+find_att([#att{name = Name} = Att | _], Name) ->
+    {ok, Att};
+find_att([_ | Rest], Name) ->
+    find_att(Rest, Name).
+
+att_md5(Att) ->
+    Md50 = couch_doc:att_foldl(
+        Att,
+        fun(Chunk, Acc) -> couch_util:md5_update(Acc, Chunk) end,
+        couch_util:md5_init()),
+    couch_util:md5_final(Md50).
+
+att_decoded_md5(Att) ->
+    Md50 = couch_doc:att_foldl_decode(
+        Att,
+        fun(Chunk, Acc) -> couch_util:md5_update(Acc, Chunk) end,
+        couch_util:md5_init()),
+    couch_util:md5_final(Md50).
+
+db_url(DbName) ->
+    iolist_to_binary([
+        "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+        ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+        "/", DbName
+    ]).
+
+replicate(Source, Target) ->
+    RepObject = {[{<<"source">>, Source}, {<<"target">>, Target}]},
+    {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_ROLE),
+    {ok, Pid} = couch_replicator:async_replicate(Rep),
+    MonRef = erlang:monitor(process, Pid),
+    receive
+        {'DOWN', MonRef, process, Pid, _} ->
+            ok
+    end.

http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2850b946/test/couch_replicator_many_leaves_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_many_leaves_tests.erl b/test/couch_replicator_many_leaves_tests.erl
new file mode 100644
index 0000000..7d0de9e
--- /dev/null
+++ b/test/couch_replicator_many_leaves_tests.erl
@@ -0,0 +1,232 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_many_leaves_tests).
+
+-include("../../../test/couchdb/couch_eunit.hrl").
+-include_lib("couchdb/couch_db.hrl").
+
+-define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
+-define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
+-define(DOCS_CONFLICTS, [
+    {<<"doc1">>, 10},
+    {<<"doc2">>, 100},
+    % a number > MaxURLlength (7000) / length(DocRevisionString)
+    {<<"doc3">>, 210}
+]).
+-define(NUM_ATTS, 2).
+-define(TIMEOUT_STOP, 1000).
+-define(TIMEOUT_EUNIT, 60).
+-define(i2l(I), integer_to_list(I)).
+-define(io2b(Io), iolist_to_binary(Io)).
+
+setup() ->
+    DbName = ?tempdb(),
+    {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
+    ok = couch_db:close(Db),
+    DbName.
+
+setup(local) ->
+    setup();
+setup(remote) ->
+    {remote, setup()};
+setup({A, B}) ->
+    {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
+    Source = setup(A),
+    Target = setup(B),
+    {Source, Target}.
+
+teardown({remote, DbName}) ->
+    teardown(DbName);
+teardown(DbName) ->
+    ok = couch_server:delete(DbName, [?ADMIN_USER]),
+    ok.
+
+teardown(_, {Source, Target}) ->
+    teardown(Source),
+    teardown(Target),
+
+    Pid = whereis(couch_server_sup),
+    erlang:monitor(process, Pid),
+    couch_server_sup:stop(),
+    receive
+        {'DOWN', _, _, Pid, _} ->
+            ok
+    after ?TIMEOUT_STOP ->
+        throw({timeout, server_stop})
+    end.
+
+
+docs_with_many_leaves_test_() ->
+    Pairs = [{local, local}, {local, remote},
+             {remote, local}, {remote, remote}],
+    {
+        "Replicate documents with many leaves",
+        {
+            foreachx,
+            fun setup/1, fun teardown/2,
+            [{Pair, fun should_populate_replicate_compact/2}
+             || Pair <- Pairs]
+        }
+    }.
+
+
+should_populate_replicate_compact({From, To}, {Source, Target}) ->
+    {lists:flatten(io_lib:format("~p -> ~p", [From, To])),
+     {inorder, [
+        should_populate_source(Source),
+        should_replicate(Source, Target),
+        should_verify_target(Source, Target),
+        should_add_attachments_to_source(Source),
+        should_replicate(Source, Target),
+        should_verify_target(Source, Target)
+     ]}}.
+
+should_populate_source({remote, Source}) ->
+    should_populate_source(Source);
+should_populate_source(Source) ->
+    {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source))}.
+
+should_replicate({remote, Source}, Target) ->
+    should_replicate(db_url(Source), Target);
+should_replicate(Source, {remote, Target}) ->
+    should_replicate(Source, db_url(Target));
+should_replicate(Source, Target) ->
+    {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
+
+should_verify_target({remote, Source}, Target) ->
+    should_verify_target(Source, Target);
+should_verify_target(Source, {remote, Target}) ->
+    should_verify_target(Source, Target);
+should_verify_target(Source, Target) ->
+    {timeout, ?TIMEOUT_EUNIT, ?_test(begin
+        {ok, SourceDb} = couch_db:open_int(Source, []),
+        {ok, TargetDb} = couch_db:open_int(Target, []),
+        verify_target(SourceDb, TargetDb, ?DOCS_CONFLICTS),
+        ok = couch_db:close(SourceDb),
+        ok = couch_db:close(TargetDb)
+    end)}.
+
+should_add_attachments_to_source({remote, Source}) ->
+    should_add_attachments_to_source(Source);
+should_add_attachments_to_source(Source) ->
+    {timeout, ?TIMEOUT_EUNIT, ?_test(begin
+        {ok, SourceDb} = couch_db:open_int(Source, []),
+        add_attachments(SourceDb, ?NUM_ATTS, ?DOCS_CONFLICTS),
+        ok = couch_db:close(SourceDb)
+    end)}.
+
+populate_db(DbName) ->
+    {ok, Db} = couch_db:open_int(DbName, []),
+    lists:foreach(
+       fun({DocId, NumConflicts}) ->
+            Value = <<"0">>,
+            Doc = #doc{
+                id = DocId,
+                body = {[ {<<"value">>, Value} ]}
+            },
+            {ok, _} = couch_db:update_doc(Db, Doc, []),
+            {ok, _} = add_doc_siblings(Db, DocId, NumConflicts)
+        end, ?DOCS_CONFLICTS),
+    couch_db:close(Db).
+
+add_doc_siblings(Db, DocId, NumLeaves) when NumLeaves > 0 ->
+    add_doc_siblings(Db, DocId, NumLeaves, [], []).
+
+add_doc_siblings(Db, _DocId, 0, AccDocs, AccRevs) ->
+    {ok, []} = couch_db:update_docs(Db, AccDocs, [], replicated_changes),
+    {ok, AccRevs};
+
+add_doc_siblings(Db, DocId, NumLeaves, AccDocs, AccRevs) ->
+    Value = ?l2b(?i2l(NumLeaves)),
+    Rev = couch_util:md5(Value),
+    Doc = #doc{
+        id = DocId,
+        revs = {1, [Rev]},
+        body = {[ {<<"value">>, Value} ]}
+    },
+    add_doc_siblings(Db, DocId, NumLeaves - 1,
+                     [Doc | AccDocs], [{1, Rev} | AccRevs]).
+
+verify_target(_SourceDb, _TargetDb, []) ->
+    ok;
+verify_target(SourceDb, TargetDb, [{DocId, NumConflicts} | Rest]) ->
+    {ok, SourceLookups} = couch_db:open_doc_revs(
+        SourceDb,
+        DocId,
+        all,
+        [conflicts, deleted_conflicts]),
+    {ok, TargetLookups} = couch_db:open_doc_revs(
+        TargetDb,
+        DocId,
+        all,
+        [conflicts, deleted_conflicts]),
+    SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
+    TargetDocs = [Doc || {ok, Doc} <- TargetLookups],
+    Total = NumConflicts + 1,
+    ?assertEqual(Total, length(TargetDocs)),
+    lists:foreach(
+        fun({SourceDoc, TargetDoc}) ->
+            SourceJson = couch_doc:to_json_obj(SourceDoc, [attachments]),
+            TargetJson = couch_doc:to_json_obj(TargetDoc, [attachments]),
+            ?assertEqual(SourceJson, TargetJson)
+        end,
+        lists:zip(SourceDocs, TargetDocs)),
+    verify_target(SourceDb, TargetDb, Rest).
+
+add_attachments(_SourceDb, _NumAtts,  []) ->
+    ok;
+add_attachments(SourceDb, NumAtts,  [{DocId, NumConflicts} | Rest]) ->
+    {ok, SourceLookups} = couch_db:open_doc_revs(SourceDb, DocId, all, []),
+    SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
+    Total = NumConflicts + 1,
+    ?assertEqual(Total, length(SourceDocs)),
+    NewDocs = lists:foldl(
+        fun(#doc{atts = Atts, revs = {Pos, [Rev | _]}} = Doc, Acc) ->
+            NewAtts = lists:foldl(fun(I, AttAcc) ->
+                AttData = crypto:rand_bytes(100),
+                NewAtt = #att{
+                    name = ?io2b(["att_", ?i2l(I), "_",
+                                  couch_doc:rev_to_str({Pos, Rev})]),
+                    type = <<"application/foobar">>,
+                    att_len = byte_size(AttData),
+                    data = AttData
+                },
+                [NewAtt | AttAcc]
+            end, [], lists:seq(1, NumAtts)),
+            [Doc#doc{atts = Atts ++ NewAtts} | Acc]
+        end,
+        [], SourceDocs),
+    {ok, UpdateResults} = couch_db:update_docs(SourceDb, NewDocs, []),
+    NewRevs = [R || {ok, R} <- UpdateResults],
+    ?assertEqual(length(NewDocs), length(NewRevs)),
+    add_attachments(SourceDb, NumAtts, Rest).
+
+db_url(DbName) ->
+    iolist_to_binary([
+        "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+        ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+        "/", DbName
+    ]).
+
+replicate(Source, Target) ->
+    RepObject = {[
+        {<<"source">>, Source},
+        {<<"target">>, Target}
+    ]},
+    {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_ROLE),
+    {ok, Pid} = couch_replicator:async_replicate(Rep),
+    MonRef = erlang:monitor(process, Pid),
+    receive
+        {'DOWN', MonRef, process, Pid, _} ->
+            ok
+    end.

http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2850b946/test/couch_replicator_missing_stubs_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_missing_stubs_tests.erl b/test/couch_replicator_missing_stubs_tests.erl
new file mode 100644
index 0000000..0c2c30d
--- /dev/null
+++ b/test/couch_replicator_missing_stubs_tests.erl
@@ -0,0 +1,260 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_missing_stubs_tests).
+
+-include("../../../test/couchdb/couch_eunit.hrl").
+-include_lib("couchdb/couch_db.hrl").
+
+-define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
+-define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
+-define(REVS_LIMIT, 3).
+-define(TIMEOUT_STOP, 1000).
+-define(TIMEOUT_EUNIT, 30).
+
+
+setup() ->
+    DbName = ?tempdb(),
+    {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
+    ok = couch_db:close(Db),
+    DbName.
+
+setup(local) ->
+    setup();
+setup(remote) ->
+    {remote, setup()};
+setup({A, B}) ->
+    {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
+    Source = setup(A),
+    Target = setup(B),
+    {Source, Target}.
+
+teardown({remote, DbName}) ->
+    teardown(DbName);
+teardown(DbName) ->
+    ok = couch_server:delete(DbName, [?ADMIN_USER]),
+    ok.
+
+teardown(_, {Source, Target}) ->
+    teardown(Source),
+    teardown(Target),
+
+    Pid = whereis(couch_server_sup),
+    erlang:monitor(process, Pid),
+    couch_server_sup:stop(),
+    receive
+        {'DOWN', _, _, Pid, _} ->
+            ok
+    after ?TIMEOUT_STOP ->
+        throw({timeout, server_stop})
+    end.
+
+
+missing_stubs_test_() ->
+    Pairs = [{local, local}, {local, remote},
+             {remote, local}, {remote, remote}],
+    {
+        "Replicate docs with missing stubs (COUCHDB-1365)",
+        {
+            foreachx,
+            fun setup/1, fun teardown/2,
+            [{Pair, fun should_replicate_docs_with_missed_att_stubs/2}
+             || Pair <- Pairs]
+        }
+    }.
+
+
+should_replicate_docs_with_missed_att_stubs({From, To}, {Source, Target}) ->
+    {lists:flatten(io_lib:format("~p -> ~p", [From, To])),
+     {inorder, [
+        should_populate_source(Source),
+        should_set_target_revs_limit(Target, ?REVS_LIMIT),
+        should_replicate(Source, Target),
+        should_compare_databases(Source, Target),
+        should_update_source_docs(Source, ?REVS_LIMIT * 2),
+        should_replicate(Source, Target),
+        should_compare_databases(Source, Target)
+     ]}}.
+
+should_populate_source({remote, Source}) ->
+    should_populate_source(Source);
+should_populate_source(Source) ->
+    {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source))}.
+
+should_replicate({remote, Source}, Target) ->
+    should_replicate(db_url(Source), Target);
+should_replicate(Source, {remote, Target}) ->
+    should_replicate(Source, db_url(Target));
+should_replicate(Source, Target) ->
+    {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
+
+should_set_target_revs_limit({remote, Target}, RevsLimit) ->
+    should_set_target_revs_limit(Target, RevsLimit);
+should_set_target_revs_limit(Target, RevsLimit) ->
+    ?_test(begin
+        {ok, Db} = couch_db:open_int(Target, [?ADMIN_USER]),
+        ?assertEqual(ok, couch_db:set_revs_limit(Db, RevsLimit)),
+        ok = couch_db:close(Db)
+    end).
+
+should_compare_databases({remote, Source}, Target) ->
+    should_compare_databases(Source, Target);
+should_compare_databases(Source, {remote, Target}) ->
+    should_compare_databases(Source, Target);
+should_compare_databases(Source, Target) ->
+    {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}.
+
+should_update_source_docs({remote, Source}, Times) ->
+    should_update_source_docs(Source, Times);
+should_update_source_docs(Source, Times) ->
+    {timeout, ?TIMEOUT_EUNIT, ?_test(update_db_docs(Source, Times))}.
+
+
+populate_db(DbName) ->
+    {ok, Db} = couch_db:open_int(DbName, []),
+    AttData = crypto:rand_bytes(6000),
+    Doc = #doc{
+        id = <<"doc1">>,
+        atts = [
+            #att{
+                name = <<"doc1_att1">>,
+                type = <<"application/foobar">>,
+                att_len = byte_size(AttData),
+                data = AttData
+            }
+        ]
+    },
+    {ok, _} = couch_db:update_doc(Db, Doc, []),
+    couch_db:close(Db).
+
+update_db_docs(DbName, Times) ->
+    {ok, Db} = couch_db:open_int(DbName, []),
+    {ok, _, _} = couch_db:enum_docs(
+        Db,
+        fun(FDI, _, Acc) -> db_fold_fun(FDI, Acc) end,
+        {DbName, Times},
+        []),
+    ok = couch_db:close(Db).
+
+db_fold_fun(FullDocInfo, {DbName, Times}) ->
+    {ok, Db} = couch_db:open_int(DbName, []),
+    {ok, Doc} = couch_db:open_doc(Db, FullDocInfo),
+    lists:foldl(
+        fun(_, {Pos, RevId}) ->
+            {ok, Db2} = couch_db:reopen(Db),
+            NewDocVersion = Doc#doc{
+                revs = {Pos, [RevId]},
+                body = {[{<<"value">>, base64:encode(crypto:rand_bytes(100))}]}
+            },
+            {ok, NewRev} = couch_db:update_doc(Db2, NewDocVersion, []),
+            NewRev
+        end,
+        {element(1, Doc#doc.revs), hd(element(2, Doc#doc.revs))},
+        lists:seq(1, Times)),
+    ok = couch_db:close(Db),
+    {ok, {DbName, Times}}.
+
+compare_dbs(Source, Target) ->
+    {ok, SourceDb} = couch_db:open_int(Source, []),
+    {ok, TargetDb} = couch_db:open_int(Target, []),
+
+    Fun = fun(FullDocInfo, _, Acc) ->
+        {ok, DocSource} = couch_db:open_doc(SourceDb, FullDocInfo,
+                                            [conflicts, deleted_conflicts]),
+        Id = DocSource#doc.id,
+
+        {ok, DocTarget} = couch_db:open_doc(TargetDb, Id,
+                                            [conflicts, deleted_conflicts]),
+        ?assertEqual(DocSource#doc.body, DocTarget#doc.body),
+
+        ?assertEqual(couch_doc:to_json_obj(DocSource, []),
+                     couch_doc:to_json_obj(DocTarget, [])),
+
+        #doc{atts = SourceAtts} = DocSource,
+        #doc{atts = TargetAtts} = DocTarget,
+        ?assertEqual(lists:sort([N || #att{name = N} <- SourceAtts]),
+                     lists:sort([N || #att{name = N} <- TargetAtts])),
+
+        lists:foreach(
+            fun(#att{name = AttName} = Att) ->
+                {ok, AttTarget} = find_att(TargetAtts, AttName),
+                SourceMd5 = att_md5(Att),
+                TargetMd5 = att_md5(AttTarget),
+                case AttName of
+                    <<"att1">> ->
+                        ?assertEqual(gzip, Att#att.encoding),
+                        ?assertEqual(gzip, AttTarget#att.encoding),
+                        DecSourceMd5 = att_decoded_md5(Att),
+                        DecTargetMd5 = att_decoded_md5(AttTarget),
+                        ?assertEqual(DecSourceMd5, DecTargetMd5);
+                    _ ->
+                        ?assertEqual(identity, Att#att.encoding),
+                        ?assertEqual(identity, AttTarget#att.encoding)
+                end,
+                ?assertEqual(SourceMd5, TargetMd5),
+                ?assert(is_integer(Att#att.disk_len)),
+                ?assert(is_integer(Att#att.att_len)),
+                ?assert(is_integer(AttTarget#att.disk_len)),
+                ?assert(is_integer(AttTarget#att.att_len)),
+                ?assertEqual(Att#att.disk_len, AttTarget#att.disk_len),
+                ?assertEqual(Att#att.att_len, AttTarget#att.att_len),
+                ?assertEqual(Att#att.type, AttTarget#att.type),
+                ?assertEqual(Att#att.md5, AttTarget#att.md5)
+            end,
+            SourceAtts),
+        {ok, Acc}
+    end,
+
+    {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
+    ok = couch_db:close(SourceDb),
+    ok = couch_db:close(TargetDb).
+
+find_att([], _Name) ->
+    nil;
+find_att([#att{name = Name} = Att | _], Name) ->
+    {ok, Att};
+find_att([_ | Rest], Name) ->
+    find_att(Rest, Name).
+
+att_md5(Att) ->
+    Md50 = couch_doc:att_foldl(
+        Att,
+        fun(Chunk, Acc) -> couch_util:md5_update(Acc, Chunk) end,
+        couch_util:md5_init()),
+    couch_util:md5_final(Md50).
+
+att_decoded_md5(Att) ->
+    Md50 = couch_doc:att_foldl_decode(
+        Att,
+        fun(Chunk, Acc) -> couch_util:md5_update(Acc, Chunk) end,
+        couch_util:md5_init()),
+    couch_util:md5_final(Md50).
+
+db_url(DbName) ->
+    iolist_to_binary([
+        "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+        ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+        "/", DbName
+    ]).
+
+replicate(Source, Target) ->
+    RepObject = {[
+        {<<"source">>, Source},
+        {<<"target">>, Target}
+    ]},
+    {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_ROLE),
+    {ok, Pid} = couch_replicator:async_replicate(Rep),
+    MonRef = erlang:monitor(process, Pid),
+    receive
+        {'DOWN', MonRef, process, Pid, _} ->
+            ok
+    end.

http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2850b946/test/couch_replicator_modules_load_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_modules_load_tests.erl b/test/couch_replicator_modules_load_tests.erl
new file mode 100644
index 0000000..cea4cc2
--- /dev/null
+++ b/test/couch_replicator_modules_load_tests.erl
@@ -0,0 +1,40 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_modules_load_tests).
+
+-include("../../../test/couchdb/couch_eunit.hrl").
+
+
+modules_load_test_() ->
+    {
+        "Verify that all modules loads",
+        should_load_modules()
+    }.
+
+
+should_load_modules() ->
+    Modules = [
+        couch_replicator_api_wrap,
+        couch_replicator_httpc,
+        couch_replicator_httpd,
+        couch_replicator_manager,
+        couch_replicator_notifier,
+        couch_replicator,
+        couch_replicator_worker,
+        couch_replicator_utils,
+        couch_replicator_job_sup
+    ],
+    [should_load_module(Mod) || Mod <- Modules].
+
+should_load_module(Mod) ->
+    {atom_to_list(Mod), ?_assertMatch({module, _}, code:load_file(Mod))}.

http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2850b946/test/couch_replicator_use_checkpoints_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_use_checkpoints_tests.erl b/test/couch_replicator_use_checkpoints_tests.erl
new file mode 100644
index 0000000..f09a235
--- /dev/null
+++ b/test/couch_replicator_use_checkpoints_tests.erl
@@ -0,0 +1,200 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_use_checkpoints_tests).
+
+-include("../../../test/couchdb/couch_eunit.hrl").
+-include_lib("couchdb/couch_db.hrl").
+
+-define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
+-define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
+-define(DOCS_COUNT, 100).
+-define(TIMEOUT_STOP, 1000).
+-define(TIMEOUT_EUNIT, 30).
+-define(i2l(I), integer_to_list(I)).
+-define(io2b(Io), iolist_to_binary(Io)).
+
+
+start(false) ->
+    fun
+        ({finished, _, {CheckpointHistory}}) ->
+            ?assertEqual([{<<"use_checkpoints">>,false}], CheckpointHistory);
+        (_) ->
+            ok
+    end;
+start(true) ->
+    fun
+        ({finished, _, {CheckpointHistory}}) ->
+            ?assertNotEqual(false, lists:keyfind(<<"session_id">>,
+                                                 1, CheckpointHistory));
+        (_) ->
+            ok
+    end.
+
+stop(_, _) ->
+    ok.
+
+setup() ->
+    DbName = ?tempdb(),
+    {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
+    ok = couch_db:close(Db),
+    DbName.
+
+setup(local) ->
+    setup();
+setup(remote) ->
+    {remote, setup()};
+setup({_, Fun, {A, B}}) ->
+    {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
+    {ok, Listener} = couch_replicator_notifier:start_link(Fun),
+    Source = setup(A),
+    Target = setup(B),
+    {Source, Target, Listener}.
+
+teardown({remote, DbName}) ->
+    teardown(DbName);
+teardown(DbName) ->
+    ok = couch_server:delete(DbName, [?ADMIN_USER]),
+    ok.
+
+teardown(_, {Source, Target, Listener}) ->
+    teardown(Source),
+    teardown(Target),
+
+    couch_replicator_notifier:stop(Listener),
+    Pid = whereis(couch_server_sup),
+    erlang:monitor(process, Pid),
+    couch_server_sup:stop(),
+    receive
+        {'DOWN', _, _, Pid, _} ->
+            ok
+    after ?TIMEOUT_STOP ->
+        throw({timeout, server_stop})
+    end.
+
+
+use_checkpoints_test_() ->
+    {
+        "Replication use_checkpoints feature tests",
+        {
+            foreachx,
+            fun start/1, fun stop/2,
+            [{UseCheckpoints, fun use_checkpoints_tests/2}
+             || UseCheckpoints <- [false, true]]
+        }
+    }.
+
+use_checkpoints_tests(UseCheckpoints, Fun) ->
+    Pairs = [{local, local}, {local, remote},
+             {remote, local}, {remote, remote}],
+    {
+        "use_checkpoints: " ++ atom_to_list(UseCheckpoints),
+        {
+            foreachx,
+            fun setup/1, fun teardown/2,
+            [{{UseCheckpoints, Fun, Pair}, fun should_test_checkpoints/2}
+             || Pair <- Pairs]
+        }
+    }.
+
+should_test_checkpoints({UseCheckpoints, _, {From, To}}, {Source, Target, _}) ->
+    should_test_checkpoints(UseCheckpoints, {From, To}, {Source, Target}).
+should_test_checkpoints(UseCheckpoints, {From, To}, {Source, Target}) ->
+    {lists:flatten(io_lib:format("~p -> ~p", [From, To])),
+     {inorder, [
+        should_populate_source(Source, ?DOCS_COUNT),
+        should_replicate(Source, Target, UseCheckpoints),
+        should_compare_databases(Source, Target)
+     ]}}.
+
+should_populate_source({remote, Source}, DocCount) ->
+    should_populate_source(Source, DocCount);
+should_populate_source(Source, DocCount) ->
+    {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source, DocCount))}.
+
+should_replicate({remote, Source}, Target, UseCheckpoints) ->
+    should_replicate(db_url(Source), Target, UseCheckpoints);
+should_replicate(Source, {remote, Target}, UseCheckpoints) ->
+    should_replicate(Source, db_url(Target), UseCheckpoints);
+should_replicate(Source, Target, UseCheckpoints) ->
+    {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target, UseCheckpoints))}.
+
+should_compare_databases({remote, Source}, Target) ->
+    should_compare_databases(Source, Target);
+should_compare_databases(Source, {remote, Target}) ->
+    should_compare_databases(Source, Target);
+should_compare_databases(Source, Target) ->
+    {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}.
+
+
+populate_db(DbName, DocCount) ->
+    {ok, Db} = couch_db:open_int(DbName, []),
+    Docs = lists:foldl(
+        fun(DocIdCounter, Acc) ->
+            Id = ?io2b(["doc", ?i2l(DocIdCounter)]),
+            Value = ?io2b(["val", ?i2l(DocIdCounter)]),
+            Doc = #doc{
+                id = Id,
+                body = {[ {<<"value">>, Value} ]}
+            },
+            [Doc | Acc]
+        end,
+        [], lists:seq(1, DocCount)),
+    {ok, _} = couch_db:update_docs(Db, Docs, []),
+    ok = couch_db:close(Db).
+
+compare_dbs(Source, Target) ->
+    {ok, SourceDb} = couch_db:open_int(Source, []),
+    {ok, TargetDb} = couch_db:open_int(Target, []),
+    Fun = fun(FullDocInfo, _, Acc) ->
+        {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo),
+        {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]),
+        DocId = couch_util:get_value(<<"_id">>, Props),
+        DocTarget = case couch_db:open_doc(TargetDb, DocId) of
+            {ok, DocT} ->
+                DocT;
+            Error ->
+                erlang:error(
+                    {assertion_failed,
+                     [{module, ?MODULE}, {line, ?LINE},
+                      {reason, lists:concat(["Error opening document '",
+                                             ?b2l(DocId), "' from target: ",
+                                             couch_util:to_list(Error)])}]})
+            end,
+        DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]),
+        ?assertEqual(DocJson, DocTargetJson),
+        {ok, Acc}
+    end,
+    {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
+    ok = couch_db:close(SourceDb),
+    ok = couch_db:close(TargetDb).
+
+db_url(DbName) ->
+    iolist_to_binary([
+        "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+        ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+        "/", DbName
+    ]).
+
+replicate(Source, Target, UseCheckpoints) ->
+    RepObject = {[
+        {<<"source">>, Source},
+        {<<"target">>, Target},
+        {<<"use_checkpoints">>, UseCheckpoints}
+    ]},
+    {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_ROLE),
+    {ok, Pid} = couch_replicator:async_replicate(Rep),
+    MonRef = erlang:monitor(process, Pid),
+    receive
+        {'DOWN', MonRef, process, Pid, _} ->
+            ok
+    end.


Mime
View raw message