Return-Path: X-Original-To: apmail-couchdb-commits-archive@www.apache.org Delivered-To: apmail-couchdb-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 6799CB449 for ; Sun, 22 Jan 2012 23:44:37 +0000 (UTC) Received: (qmail 24629 invoked by uid 500); 22 Jan 2012 23:44:37 -0000 Delivered-To: apmail-couchdb-commits-archive@couchdb.apache.org Received: (qmail 24539 invoked by uid 500); 22 Jan 2012 23:44:37 -0000 Mailing-List: contact commits-help@couchdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@couchdb.apache.org Delivered-To: mailing list commits@couchdb.apache.org Received: (qmail 24462 invoked by uid 99); 22 Jan 2012 23:44:37 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 22 Jan 2012 23:44:37 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.114] (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 22 Jan 2012 23:44:35 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 9DEE53190F9; Sun, 22 Jan 2012 23:43:14 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: fdmanana@apache.org To: commits@couchdb.apache.org X-Mailer: ASF-Git Admin Mailer Subject: [40/50] git commit: Improve test 05-replication-many-leaves.t Message-Id: <20120122234314.9DEE53190F9@tyr.zones.apache.org> Date: Sun, 22 Jan 2012 23:43:14 +0000 (UTC) Improve test 05-replication-many-leaves.t The test now includes attachments to verify that the incremental attachment replication code path succeeds when there are many leaf revisions of a document with attachments. Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/75b6e09b Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/75b6e09b Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/75b6e09b Branch: refs/heads/COUCHDB-1342 Commit: 75b6e09b5bfa9b2f0022d69cf53d41b5757f1146 Parents: 32fb9f8 Author: Filipe David Borba Manana Authored: Wed Jan 4 13:47:27 2012 +0000 Committer: Filipe David Borba Manana Committed: Wed Jan 4 13:47:27 2012 +0000 ---------------------------------------------------------------------- .../test/05-replication-many-leaves.t | 148 +++++++++++---- 1 files changed, 112 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb/blob/75b6e09b/src/couch_replicator/test/05-replication-many-leaves.t ---------------------------------------------------------------------- diff --git a/src/couch_replicator/test/05-replication-many-leaves.t b/src/couch_replicator/test/05-replication-many-leaves.t index 52d2023..22c7df8 100755 --- a/src/couch_replicator/test/05-replication-many-leaves.t +++ b/src/couch_replicator/test/05-replication-many-leaves.t @@ -32,6 +32,17 @@ meta = [] }). +-record(att, { + name, + type, + att_len, + disk_len, + md5= <<>>, + revpos=0, + data, + encoding=identity +}). + -define(b2l(B), binary_to_list(B)). -define(l2b(L), list_to_binary(L)). -define(i2l(I), integer_to_list(I)). @@ -43,15 +54,15 @@ target_db_name() -> <<"couch_test_rep_db_b">>. doc_ids() -> [<<"doc1">>, <<"doc2">>, <<"doc3">>]. -doc_num_conflicts(<<"doc1">>) -> 100; -doc_num_conflicts(<<"doc2">>) -> 200; -doc_num_conflicts(<<"doc3">>) -> 550. +doc_num_conflicts(<<"doc1">>) -> 10; +doc_num_conflicts(<<"doc2">>) -> 100; +doc_num_conflicts(<<"doc3">>) -> 286. main(_) -> test_util:init_code_path(), - etap:plan(16), + etap:plan(56), case (catch test()) of ok -> etap:end_tests(); @@ -74,13 +85,12 @@ test() -> {{remote, source_db_name()}, {remote, (target_db_name())}} ], - {ok, SourceDb} = create_db(source_db_name()), - etap:diag("Populating source database"), - {ok, DocRevs} = populate_db(SourceDb), - ok = couch_db:close(SourceDb), - lists:foreach( fun({Source, Target}) -> + {ok, SourceDb} = create_db(source_db_name()), + etap:diag("Populating source database"), + {ok, DocRevs} = populate_db(SourceDb), + ok = couch_db:close(SourceDb), etap:diag("Creating target database"), {ok, TargetDb} = create_db(target_db_name()), @@ -88,17 +98,31 @@ test() -> etap:diag("Triggering replication"), replicate(Source, Target), etap:diag("Replication finished, comparing source and target databases"), + {ok, SourceDb2} = couch_db:open_int(source_db_name(), []), {ok, TargetDb2} = couch_db:open_int(target_db_name(), []), - verify_target(TargetDb2, DocRevs), + verify_target(SourceDb2, TargetDb2, DocRevs), + ok = couch_db:close(SourceDb2), ok = couch_db:close(TargetDb2), - etap:diag("Deleting target database"), + {ok, SourceDb3} = couch_db:open_int(source_db_name(), []), + {ok, DocRevs2} = add_attachments(SourceDb3, DocRevs, 2), + ok = couch_db:close(SourceDb3), + etap:diag("Triggering replication again"), + replicate(Source, Target), + etap:diag("Replication finished, comparing source and target databases"), + {ok, SourceDb4} = couch_db:open_int(source_db_name(), []), + {ok, TargetDb4} = couch_db:open_int(target_db_name(), []), + verify_target(SourceDb4, TargetDb4, DocRevs2), + ok = couch_db:close(SourceDb4), + ok = couch_db:close(TargetDb4), + + etap:diag("Deleting source and target databases"), delete_db(TargetDb), + delete_db(SourceDb), ok = timer:sleep(1000) end, Pairs), - delete_db(SourceDb), couch_server_sup:stop(), ok. @@ -112,44 +136,48 @@ populate_db(Db) -> body = {[ {<<"value">>, Value} ]} }, {ok, Rev} = couch_db:update_doc(Db, Doc, []), - {ok, RevsDict} = add_doc_siblings(Db, DocId, doc_num_conflicts(DocId)), - RevsDict2 = dict:store(Rev, Value, RevsDict), - dict:store(DocId, RevsDict2, Acc) + {ok, DocRevs} = add_doc_siblings(Db, DocId, doc_num_conflicts(DocId)), + dict:store(DocId, [Rev | DocRevs], Acc) end, dict:new(), doc_ids()), {ok, dict:to_list(DocRevsDict)}. add_doc_siblings(Db, DocId, NumLeaves) when NumLeaves > 0 -> - add_doc_siblings(Db, DocId, NumLeaves, [], dict:new()). + add_doc_siblings(Db, DocId, NumLeaves, [], []). -add_doc_siblings(Db, _DocId, 0, AccDocs, RevsDict) -> +add_doc_siblings(Db, _DocId, 0, AccDocs, AccRevs) -> {ok, []} = couch_db:update_docs(Db, AccDocs, [], replicated_changes), - {ok, RevsDict}; + {ok, AccRevs}; -add_doc_siblings(Db, DocId, NumLeaves, AccDocs, RevsDict) -> +add_doc_siblings(Db, DocId, NumLeaves, AccDocs, AccRevs) -> Value = list_to_binary(integer_to_list(NumLeaves)), Rev = couch_util:md5(Value), - RevsDict2 = dict:store({1, Rev}, Value, RevsDict), Doc = #doc{ id = DocId, revs = {1, [Rev]}, body = {[ {<<"value">>, Value} ]} }, - add_doc_siblings(Db, DocId, NumLeaves - 1, [Doc | AccDocs], RevsDict2). + add_doc_siblings(Db, DocId, NumLeaves - 1, [Doc | AccDocs], [{1, Rev} | AccRevs]). -verify_target(_TargetDb, []) -> +verify_target(_SourceDb, _TargetDb, []) -> ok; -verify_target(TargetDb, [{DocId, RevsDict} | Rest]) -> +verify_target(SourceDb, TargetDb, [{DocId, RevList} | Rest]) -> {ok, Lookups} = couch_db:open_doc_revs( TargetDb, DocId, - [R || {R, _} <- dict:to_list(RevsDict)], - [ejson_body]), + RevList, + []), Docs = [Doc || {ok, Doc} <- Lookups], + {ok, SourceLookups} = couch_db:open_doc_revs( + SourceDb, + DocId, + RevList, + []), + SourceDocs = [Doc || {ok, Doc} <- SourceLookups], Total = doc_num_conflicts(DocId) + 1, etap:is( length(Docs), @@ -157,21 +185,69 @@ verify_target(TargetDb, [{DocId, RevsDict} | Rest]) -> "Target has " ++ ?i2l(Total) ++ " leaf revisions of document " ++ ?b2l(DocId)), etap:diag("Verifying all revisions of document " ++ ?b2l(DocId)), lists:foreach( - fun(#doc{revs = {Pos, [RevId]}, body = {Body}}) -> - Rev = {Pos, RevId}, - {ok, Value} = dict:find(Rev, RevsDict), - case couch_util:get_value(<<"value">>, Body) of - Value -> + fun({#doc{id = Id, revs = Revs} = TgtDoc, #doc{id = Id, revs = Revs} = SrcDoc}) -> + SourceJson = couch_doc:to_json_obj(SrcDoc, [attachments, conflicts]), + TargetJson = couch_doc:to_json_obj(TgtDoc, [attachments, conflicts]), + case TargetJson of + SourceJson -> ok; - Other -> + _ -> + {Pos, [Rev | _]} = Revs, etap:bail("Wrong value for revision " ++ - ?b2l(couch_doc:rev_to_str(Rev)) ++ " of document " ++ - ?b2l(DocId) ++ ". Expected `" ++ couch_util:to_list(Value) ++ - "`, got `" ++ couch_util:to_list(Other) ++ "`") + ?b2l(couch_doc:rev_to_str({Pos, Rev})) ++ + " of document " ++ ?b2l(DocId)) end end, - Docs), - verify_target(TargetDb, Rest). + lists:zip(Docs, SourceDocs)), + verify_target(SourceDb, TargetDb, Rest). + + +add_attachments(Source, DocIdRevs, NumAtts) -> + add_attachments(Source, DocIdRevs, NumAtts, []). + +add_attachments(_SourceDb, [], _NumAtts, Acc) -> + {ok, Acc}; + +add_attachments(SourceDb, [{DocId, RevList} | Rest], NumAtts, IdRevsAcc) -> + {ok, SourceLookups} = couch_db:open_doc_revs( + SourceDb, + DocId, + RevList, + []), + SourceDocs = [Doc || {ok, Doc} <- SourceLookups], + Total = doc_num_conflicts(DocId) + 1, + etap:is( + length(SourceDocs), + Total, + "Source still has " ++ ?i2l(Total) ++ + " leaf revisions of document " ++ ?b2l(DocId)), + etap:diag("Adding " ++ ?i2l(NumAtts) ++ + " attachments to each revision of the document " ++ ?b2l(DocId)), + NewDocs = lists:foldl( + fun(#doc{atts = Atts, revs = {Pos, [Rev | _]}} = Doc, Acc) -> + NewAtts = lists:foldl( + fun(I, AttAcc) -> + AttData = crypto:rand_bytes(100), + NewAtt = #att{ + name = iolist_to_binary( + ["att_", ?i2l(I), "_", couch_doc:rev_to_str({Pos, Rev})]), + type = <<"application/foobar">>, + att_len = byte_size(AttData), + data = AttData + }, + [NewAtt | AttAcc] + end, + [], lists:seq(1, NumAtts)), + [Doc#doc{atts = Atts ++ NewAtts} | Acc] + end, + [], SourceDocs), + {ok, UpdateResults} = couch_db:update_docs(SourceDb, NewDocs, []), + NewRevs = [R || {ok, R} <- UpdateResults], + etap:is( + length(NewRevs), + length(NewDocs), + "Document revisions updated with " ++ ?i2l(NumAtts) ++ " attachments"), + add_attachments(SourceDb, Rest, NumAtts, [{DocId, NewRevs} | IdRevsAcc]). db_url(DbName) ->