couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From fdman...@apache.org
Subject svn commit: r966765 - in /couchdb/branches/new_replicator: share/www/script/test/new_replication.js src/couchdb/couch_api_wrap.erl src/couchdb/couch_doc.erl
Date Thu, 22 Jul 2010 17:33:04 GMT
Author: fdmanana
Date: Thu Jul 22 17:33:04 2010
New Revision: 966765

URL: http://svn.apache.org/viewvc?rev=966765&view=rev
Log:
Updated the new replicator test case to test for replicated attachments and correct an issue
with attachment replication when doing a replication by doc IDs list.

Modified:
    couchdb/branches/new_replicator/share/www/script/test/new_replication.js
    couchdb/branches/new_replicator/src/couchdb/couch_api_wrap.erl
    couchdb/branches/new_replicator/src/couchdb/couch_doc.erl

Modified: couchdb/branches/new_replicator/share/www/script/test/new_replication.js
URL: http://svn.apache.org/viewvc/couchdb/branches/new_replicator/share/www/script/test/new_replication.js?rev=966765&r1=966764&r2=966765&view=diff
==============================================================================
--- couchdb/branches/new_replicator/share/www/script/test/new_replication.js (original)
+++ couchdb/branches/new_replicator/share/www/script/test/new_replication.js Thu Jul 22 17:33:04
2010
@@ -37,6 +37,12 @@ couchTests.new_replication = function(de
     }
   ];
 
+  var att1_data = CouchDB.request("GET", "/_utils/script/test/lorem.txt");
+  att1_data = att1_data.responseText;
+
+  var att2_data = CouchDB.request("GET", "/_utils/script/test/lorem_b64.txt");
+  att2_data = att2_data.responseText;
+
   var sourceInfo, targetInfo;
   var docs, doc, copy;
   var repResult;
@@ -58,6 +64,47 @@ couchTests.new_replication = function(de
   }
 
 
+  function addAtt(db, doc, attName, attData, type) {
+    var uri = "/" + db.name + "/" + encodeURIComponent(doc._id) + "/" + attName;
+
+    if (doc._rev) {
+      uri += "?rev=" + doc._rev;
+    }
+
+    var xhr = CouchDB.request("PUT", uri, {
+      headers: {
+        "Content-Type": type
+      },
+      body: attData
+    });
+
+    T(xhr.status === 201);
+    doc._rev = JSON.parse(xhr.responseText).rev;
+  }
+
+
+  function compareObjects(o1, o2) {
+    for (var p in o1) {
+      if (o1[p] === null && o2[p] !== null) {
+        return false;
+      } else if (typeof o1[p] === "object") {
+        if ((typeof o2[p] !== "object") || o2[p] === null) {
+          return false;
+        }
+        if (!arguments.callee(o1[p], o2[p])) {
+          return false;
+        }
+      } else {
+        if (o1[p] !== o2[p]) {
+          return false;
+        }
+      }
+    }
+    return true;
+  }
+
+
+
   // test simple replications (not continuous, not filtered), including
   // conflict creation
   docs = makeDocs(1, 21);
@@ -71,6 +118,11 @@ couchTests.new_replication = function(de
     populateDb(sourceDb, docs);
     populateDb(targetDb, []);
 
+    // add some attachments
+    for (j = 10; j < 15; j++) {
+      addAtt(sourceDb, docs[j], "readme.txt", att1_data, "text/plain");
+    }
+
     repResult = CouchDB.new_replicate(dbPairs[i].source, dbPairs[i].target);
     T(repResult.ok === true);
 
@@ -78,7 +130,6 @@ couchTests.new_replication = function(de
     targetInfo = targetDb.info();
 
     T(sourceInfo.doc_count === targetInfo.doc_count);
-    T(sourceInfo.update_seq === targetInfo.update_seq);
 
     T(typeof repResult.session_id === "string");
     T(repResult.source_last_seq === sourceInfo.update_seq);
@@ -101,19 +152,38 @@ couchTests.new_replication = function(de
       copy = targetDb.open(doc._id);
 
       T(copy !== null);
-      for (var p in doc) {
-        T(copy[p] === doc[p]);
+      T(compareObjects(doc, copy) === true);
+
+      if (j >= 10 && j < 15) {
+        var atts = copy._attachments;
+        T(typeof atts === "object");
+        T(typeof atts["readme.txt"] === "object");
+        T(atts["readme.txt"].revpos === 2);
+        T(atts["readme.txt"].content_type.indexOf("text/plain") === 0);
+        T(atts["readme.txt"].stub === true);
+
+        var att_copy = CouchDB.request(
+          "GET", "/" + targetDb.name + "/" + copy._id + "/readme.txt"
+        ).responseText;
+        T(att_copy.length === att1_data.length);
+        T(att_copy === att1_data);
       }
     }
 
 
-    // add one more doc to source and replicate again
+    // add one more doc to source, more attachments to some existing docs
+    // and replicate again
     var newDoc = {
       _id: "foo666",
       value: "d"
     };
     T(sourceDb.save(newDoc).ok);
 
+    // add some more attachments
+    for (j = 10; j < 15; j++) {
+      addAtt(sourceDb, docs[j], "data.dat", att2_data, "application/binary");
+    }
+
     repResult = CouchDB.new_replicate(dbPairs[i].source, dbPairs[i].target);
     T(repResult.ok === true);
 
@@ -121,7 +191,6 @@ couchTests.new_replication = function(de
     targetInfo = targetDb.info();
 
     T(sourceInfo.doc_count === targetInfo.doc_count);
-    T(sourceInfo.update_seq === targetInfo.update_seq);
 
     T(typeof repResult.session_id === "string");
     T(repResult.source_last_seq === sourceInfo.update_seq);
@@ -130,13 +199,13 @@ couchTests.new_replication = function(de
     T(repResult.history[0].session_id === repResult.session_id);
     T(typeof repResult.history[0].start_time === "string");
     T(typeof repResult.history[0].end_time === "string");
-    T(repResult.history[0].start_last_seq === (sourceInfo.update_seq - 1));
+    T(repResult.history[0].start_last_seq === (sourceInfo.update_seq - 6));
     T(repResult.history[0].end_last_seq === sourceInfo.update_seq);
     T(repResult.history[0].recorded_seq === sourceInfo.update_seq);
-    T(repResult.history[0].missing_checked === 1);
-    T(repResult.history[0].missing_found === 1);
-    T(repResult.history[0].docs_read === 1);
-    T(repResult.history[0].docs_written === 1);
+    T(repResult.history[0].missing_checked === 6);
+    T(repResult.history[0].missing_found === 6);
+    T(repResult.history[0].docs_read === 6);
+    T(repResult.history[0].docs_written === 6);
     T(repResult.history[0].doc_write_failures === 0);
 
     copy = targetDb.open(newDoc._id);
@@ -144,6 +213,38 @@ couchTests.new_replication = function(de
     T(copy._id === newDoc._id);
     T(copy.value === newDoc.value);
 
+    for (j = 10; j < 15; j++) {
+      doc = docs[j];
+      copy = targetDb.open(doc._id);
+
+      T(copy !== null);
+      T(compareObjects(doc, copy) === true);
+
+      var atts = copy._attachments;
+      T(typeof atts === "object");
+      T(typeof atts["readme.txt"] === "object");
+      T(atts["readme.txt"].revpos === 2);
+      T(atts["readme.txt"].content_type.indexOf("text/plain") === 0);
+      T(atts["readme.txt"].stub === true);
+
+      var att1_copy = CouchDB.request(
+        "GET", "/" + targetDb.name + "/" + copy._id + "/readme.txt"
+      ).responseText;
+      T(att1_copy.length === att1_data.length);
+      T(att1_copy === att1_data);
+
+      T(typeof atts["data.dat"] === "object");
+      T(atts["data.dat"].revpos === 3);
+      T(atts["data.dat"].content_type.indexOf("application/binary") === 0);
+      T(atts["data.dat"].stub === true);
+
+      var att2_copy = CouchDB.request(
+        "GET", "/" + targetDb.name + "/" + copy._id + "/data.dat"
+      ).responseText;
+      T(att2_copy.length === att2_data.length);
+      T(att2_copy === att2_data);
+    }
+
     // test deletion is replicated
     doc = sourceDb.open(docs[1]._id);
     T(sourceDb.deleteDoc(doc).ok);
@@ -155,7 +256,6 @@ couchTests.new_replication = function(de
     targetInfo = targetDb.info();
 
     T(sourceInfo.doc_count === targetInfo.doc_count);
-    T(sourceInfo.update_seq === targetInfo.update_seq);
     T(sourceInfo.doc_del_count === targetInfo.doc_del_count);
     T(targetInfo.doc_del_count === 1);
 
@@ -173,10 +273,10 @@ couchTests.new_replication = function(de
     copy = targetDb.open(docs[1]._id);
     T(copy === null);
 
-    var changes = targetDb.changes({since: sourceInfo.update_seq - 1});
-    T(changes.results[0].id === docs[1]._id);
-    T(changes.results[0].seq === sourceInfo.update_seq);
-    T(changes.results[0].deleted === true);
+    var changes = targetDb.changes({since: 0});
+    var idx = changes.results.length - 1;
+    T(changes.results[idx].id === docs[1]._id);
+    T(changes.results[idx].deleted === true);
 
     // test conflict
     doc = sourceDb.open(docs[0]._id);
@@ -194,7 +294,6 @@ couchTests.new_replication = function(de
     targetInfo = targetDb.info();
 
     T(sourceInfo.doc_count === targetInfo.doc_count);
-    T(sourceInfo.update_seq === (targetInfo.update_seq - 1));
 
     T(repResult.history instanceof Array);
     T(repResult.history.length === 4);
@@ -225,7 +324,6 @@ couchTests.new_replication = function(de
     targetInfo = targetDb.info();
 
     T(sourceInfo.doc_count === targetInfo.doc_count);
-    T(sourceInfo.update_seq === (targetInfo.update_seq - 1));
 
     T(repResult.history instanceof Array);
     T(repResult.history.length === 5);
@@ -259,7 +357,6 @@ couchTests.new_replication = function(de
     targetInfo = targetDb.info();
 
     T(sourceInfo.doc_count === targetInfo.doc_count);
-    T(sourceInfo.update_seq === (targetInfo.update_seq - 2));
 
     T(repResult.history instanceof Array);
     T(repResult.history.length === 6);
@@ -340,9 +437,7 @@ couchTests.new_replication = function(de
       if ((doc.integer && (doc.integer % 2 === 0)) || (doc.string === "7")) {
 
         T(copy !== null);
-        for (var p in doc) {
-          T(copy[p] === doc[p]);
-        }
+        T(compareObjects(doc, copy) === true);
       } else {
         T(copy === null);
       }
@@ -390,9 +485,7 @@ couchTests.new_replication = function(de
       if (doc.integer && (doc.integer % 2 === 0)) {
 
         T(copy !== null);
-        for (var p in doc) {
-          T(copy[p] === doc[p]);
-        }
+        T(compareObjects(doc, copy) === true);
       } else {
         T(copy === null);
       }
@@ -466,9 +559,6 @@ couchTests.new_replication = function(de
       T(repResult.docs_written === total);
       T(repResult.doc_write_failures === 0);
 
-      targetInfo = targetDb.info();
-      T(targetInfo.doc_count === total);
-
       for (k = 0; k < doc_ids.length; k++) {
         id = doc_ids[k];
         doc = sourceDb.open(id);
@@ -480,12 +570,26 @@ couchTests.new_replication = function(de
         } else {
           T(doc !== null);
           T(copy !== null);
-          for (var p in doc) {
-            T(copy[p] === doc[p]);
-          }
+          T(compareObjects(doc, copy) === true);
         }
       }
 
+      // be absolutely sure that other docs were not replicated
+      for (k = 0; k < docs.length; k++) {
+        id = docs[k]._id;
+        doc = targetDb.open(id);
+
+        if (doc_ids.indexOf(id) >= 0) {
+            T(doc !== null);
+        } else {
+            T(doc === null);
+        }
+      }
+
+      targetInfo = targetDb.info();
+      T(targetInfo.doc_count === total);
+
+
       // add more docs throught replication by doc IDs
       after_doc_ids = target_doc_ids[j].after;
       after_num_inexistent_docs = 0;
@@ -515,9 +619,6 @@ couchTests.new_replication = function(de
       T(repResult.docs_written === after_total);
       T(repResult.doc_write_failures === 0);
 
-      targetInfo = targetDb.info();
-      T(targetInfo.doc_count === (total + after_total));
-
       for (k = 0; k < after_doc_ids.length; k++) {
         id = after_doc_ids[k];
         doc = sourceDb.open(id);
@@ -529,18 +630,34 @@ couchTests.new_replication = function(de
         } else {
           T(doc !== null);
           T(copy !== null);
-          for (var p in doc) {
-            T(copy[p] === doc[p]);
-          }
+          T(compareObjects(doc, copy) === true);
         }
       }
 
+      // be absolutely sure that other docs were not replicated
+      for (k = 0; k < docs.length; k++) {
+        id = docs[k]._id;
+        doc = targetDb.open(id);
+
+        if ((doc_ids.indexOf(id) >= 0) || (after_doc_ids.indexOf(id) >= 0)) {
+            T(doc !== null);
+        } else {
+            T(doc === null);
+        }
+      }
+
+      targetInfo = targetDb.info();
+      T(targetInfo.doc_count === (total + after_total));
+
+
       // replicate again the same doc after updated on source (no conflict)
       id = target_doc_ids[j].conflict_id;
       doc = sourceDb.open(id);
       T(doc !== null);
-      doc.integer += 100;
+      doc.integer = 666;
       T(sourceDb.save(doc).ok);
+      addAtt(sourceDb, doc, "readme.txt", att1_data, "text/plain");
+      addAtt(sourceDb, doc, "data.dat", att2_data, "application/binary");
 
       repResult = CouchDB.new_replicate(
         dbPairs[i].source,
@@ -559,9 +676,35 @@ couchTests.new_replication = function(de
 
       copy = targetDb.open(id, {conflicts: true});
 
-      T(copy._rev.indexOf("2-") === 0);
+      T(copy.integer === 666);
+      T(copy._rev.indexOf("4-") === 0);
       T(typeof copy._conflicts === "undefined");
 
+      var atts = copy._attachments;
+      T(typeof atts === "object");
+      T(typeof atts["readme.txt"] === "object");
+      T(atts["readme.txt"].revpos === 3);
+      T(atts["readme.txt"].content_type.indexOf("text/plain") === 0);
+      T(atts["readme.txt"].stub === true);
+
+      var att1_copy = CouchDB.request(
+        "GET", "/" + targetDb.name + "/" + copy._id + "/readme.txt"
+      ).responseText;
+      T(att1_copy.length === att1_data.length);
+      T(att1_copy === att1_data);
+
+      T(typeof atts["data.dat"] === "object");
+      T(atts["data.dat"].revpos === 4);
+      T(atts["data.dat"].content_type.indexOf("application/binary") === 0);
+      T(atts["data.dat"].stub === true);
+
+      var att2_copy = CouchDB.request(
+        "GET", "/" + targetDb.name + "/" + copy._id + "/data.dat"
+      ).responseText;
+      T(att2_copy.length === att2_data.length);
+      T(att2_copy === att2_data);
+
+
       // generate a conflict throught replication by doc IDs
       id = target_doc_ids[j].conflict_id;
       doc = sourceDb.open(id);
@@ -590,10 +733,10 @@ couchTests.new_replication = function(de
 
       copy = targetDb.open(id, {conflicts: true});
 
-      T(copy._rev.indexOf("3-") === 0);
+      T(copy._rev.indexOf("5-") === 0);
       T(copy._conflicts instanceof Array);
       T(copy._conflicts.length === 1);
-      T(copy._conflicts[0].indexOf("3-") === 0);
+      T(copy._conflicts[0].indexOf("5-") === 0);
     }
   }
 

Modified: couchdb/branches/new_replicator/src/couchdb/couch_api_wrap.erl
URL: http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_api_wrap.erl?rev=966765&r1=966764&r2=966765&view=diff
==============================================================================
--- couchdb/branches/new_replicator/src/couchdb/couch_api_wrap.erl (original)
+++ couchdb/branches/new_replicator/src/couchdb/couch_api_wrap.erl Thu Jul 22 17:33:04 2010
@@ -37,10 +37,10 @@
     db_open/3,
     db_close/1,
     get_db_info/1,
-    open_doc/3,
     update_doc/3,
     ensure_full_commit/1,
     get_missing_revs/2,
+    open_doc/3,
     open_doc/4,
     open_doc_revs/6,
     update_doc/4,
@@ -94,27 +94,6 @@ get_db_info(Db) ->
     couch_db:get_db_info(Db).
 
 
-open_doc(#httpdb{url=Url, oauth=OAuth, headers=Headers}, DocId, Options) ->
-    Url2 = Url ++ couch_util:url_encode(DocId),
-    QArgs = options_to_query_args(Options, []),
-    Headers2 = oauth_header(Url2, QArgs, get, OAuth) ++ Headers,
-    #url{host=Host, port=Port} = ibrowse_lib:parse_url(Url),
-    {ok, Worker} = ibrowse:spawn_link_worker_process(Host,Port),
-    try ibrowse:send_req_direct(Worker, Url2 ++ query_args_to_string(QArgs, []), 
-            Headers2, get, [], [ 
-            {response_format,binary}
-            ], infinity) of
-    {ok, "200", _RespHeaders, Body} ->
-        {ok, couch_doc:from_json_obj(?JSON_DECODE(Body))};
-    {ok, "404", _RespHeaders, _Body} ->
-        {not_found, missing}
-    after
-        catch ibrowse:stop_worker_process(Worker)
-    end;
-open_doc(Db, DocId, Options) ->
-    couch_db:open_doc(Db, DocId, Options).
-
-
 update_doc(Db, Doc, Options) ->
     update_doc(Db,Doc,Options,interactive_edit).
 
@@ -203,8 +182,10 @@ open_doc_revs(Db, Id, Revs, Options, Fun
     {ok, Results} = couch_db:open_doc_revs(Db, Id, Revs, Options),
     {ok, lists:foldl(Fun, Acc, Results)}.
 
+open_doc(Db, Id, Options, Fun) ->
+    Fun(open_doc(Db, Id, Options)).
 
-open_doc(#httpdb{} = HttpDb, Id, Options, Fun) ->
+open_doc(#httpdb{} = HttpDb, Id, Options) ->
     #httpdb{url=Url, oauth=OAuth, headers=Headers} = HttpDb,
     QArgs = [
         {"attachments", "true"},
@@ -229,39 +210,48 @@ open_doc(#httpdb{} = HttpDb, Id, Options
                 [{response_format, binary}, {stream_to, {self(), once}}],
                 infinity),
 
-            DataFun = fun() -> stream_data_self(ReqId) end,
             receive
-            {ibrowse_async_headers, ReqId, "200", RespHeaders} ->
-                case couch_util:get_value("Content-Type", RespHeaders) of
-                ("multipart/related;" ++ _) = CType ->
-                     {ok, Doc} = couch_doc:doc_from_multi_part_stream(
-                         CType, DataFun);
+            {ibrowse_async_headers, ReqId, Code, RespHeaders} ->
+                CType = couch_util:get_value("Content-Type", RespHeaders),
+                Self ! {self(), CType},
+                case CType of
                 "application/json" ->
-                     Doc = couch_doc:from_json_obj(
-                         json_stream_parse:to_ejson(DataFun))
-                end,
-                receive
-                {get_doc, From} ->
-                    From ! {doc, Doc}
-                end;
-            {ibrowse_async_headers, ReqId, _ErrorCode, _RespHeaders} ->
-                receive
-                {get_doc, From} ->
-                    From ! {error, json_stream_parse:to_ejson(DataFun)}
+                    receive
+                    {get, From} ->
+                        EJson = json_stream_parse:to_ejson(
+                            fun() -> stream_data_self(ReqId) end),
+                        case Code of
+                        "200" ->
+                            Doc = couch_doc:from_json_obj(EJson),
+                            From ! {data, self(), Doc};
+                        _ErrorCode ->
+                            From ! {data, self(), EJson}
+                         end
+                    end;
+                "multipart/related;" ++ _ ->
+                    couch_httpd:parse_multipart_request(
+                        CType,
+                        fun() -> stream_data_self(ReqId) end,
+                        fun(Ev)-> couch_doc:mp_parse_doc(Ev, []) end)
                 end
             end,
             catch ibrowse:stop_worker_process(Worker),
             unlink(Self)
         end),
-    Streamer ! {get_doc, self()},
     receive
-    {doc, Doc} ->
-        Fun({ok, Doc});
-    {error, Error} ->
-        Fun(Error)
+    {Streamer, "application/json"} ->
+        Streamer ! {get, self()},
+        receive
+        {data, Streamer, #doc{} = Doc} ->
+            {ok, Doc};
+        {data, Streamer, Error} ->
+            Error
+        end;
+    {Streamer, "multipart/related;" ++ _} ->
+        couch_doc:doc_from_multi_part_stream(Streamer)
     end;
-open_doc(Db, Id, Options, Fun) ->
-    Fun(couch_db:open_doc(Db, Id, Options)).
+open_doc(Db, Id, Options) ->
+    couch_db:open_doc(Db, Id, Options).
 
 
 update_doc(#httpdb{} = HttpDb, Doc, Options, Type) ->

Modified: couchdb/branches/new_replicator/src/couchdb/couch_doc.erl
URL: http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_doc.erl?rev=966765&r1=966764&r2=966765&view=diff
==============================================================================
--- couchdb/branches/new_replicator/src/couchdb/couch_doc.erl (original)
+++ couchdb/branches/new_replicator/src/couchdb/couch_doc.erl Thu Jul 22 17:33:04 2010
@@ -16,8 +16,9 @@
 -export([att_foldl/3,att_foldl_decode/3,get_validate_doc_fun/1]).
 -export([from_json_obj/1,to_json_obj/2,has_stubs/1, merge_stubs/2]).
 -export([validate_docid/1]).
--export([doc_from_multi_part_stream/2]).
+-export([doc_from_multi_part_stream/1, doc_from_multi_part_stream/2]).
 -export([doc_to_multi_part_stream/5, len_doc_to_multi_part_stream/4]).
+-export([mp_parse_doc/2]).
 
 -include("couch_db.hrl").
 
@@ -441,13 +442,7 @@ atts_to_mp([Att | RestAtts], Boundary, W
     atts_to_mp(RestAtts, Boundary, WriteFun, SendEncodedAtts).
 
 
-doc_from_multi_part_stream(ContentType, DataFun) ->
-    Self = self(),
-    Parser = spawn_link(fun() ->
-        couch_httpd:parse_multipart_request(ContentType, DataFun,
-                fun(Next)-> mp_parse_doc(Next, []) end),
-        unlink(Self)
-        end),
+doc_from_multi_part_stream(Parser) ->
     Parser ! {get_doc_bytes, self()},
     receive 
     {doc_bytes, DocBytes} ->
@@ -467,6 +462,15 @@ doc_from_multi_part_stream(ContentType, 
         {ok, Doc#doc{atts=Atts2}}
     end.
 
+doc_from_multi_part_stream(ContentType, DataFun) ->
+    Self = self(),
+    Parser = spawn_link(fun() ->
+        couch_httpd:parse_multipart_request(ContentType, DataFun,
+                fun(Next)-> mp_parse_doc(Next, []) end),
+        unlink(Self)
+        end),
+    doc_from_multi_part_stream(Parser).
+
 mp_parse_doc({headers, H}, []) ->
     case couch_util:get_value("content-type", H) of
     {"application/json", _} ->



Mime
View raw message