Mailing-List: contact dev-help@couchdb.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@couchdb.apache.org
Received-SPF: pass (nike.apache.org: domain of fdmanana@gmail.com designates
 209.85.210.46 as permitted sender)
MIME-Version: 1.0
Sender: fdmanana@gmail.com
In-Reply-To: 
 <CAJ_m3YBUOmZoGG1TTpiSum+83pFpsoiS670OBtjGoP4ofAeC3w@mail.gmail.com>
References: 
 <CAJ_m3YDr_G34C5SK=61hbkNWCP52=F46odGHn7wam7eU3cBktQ@mail.gmail.com>
	<CAL3q7H7PzRibp84usbU2ybSY5UOeFH+e8fVT8C+N3ZE3SFT9kg@mail.gmail.com>
	<CAJ_m3YBm6fu1EP8fZ9do8gj0kqgph04oR1-0_k1spMg4kF=-HA@mail.gmail.com>
	<CAJ_m3YBUOmZoGG1TTpiSum+83pFpsoiS670OBtjGoP4ofAeC3w@mail.gmail.com>
Date: Wed, 10 Aug 2011 18:15:31 -0700
Message-ID: 
 <CAL3q7H5XS1e61dkUxY8thDy-L7FZgCbfNkwsAFqGAV2kD7xpug@mail.gmail.com>
Subject: Re: Futon Test Suite
From: Filipe David Manana <fdmanana@apache.org>
To: dev@couchdb.apache.org
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: quoted-printable

On Wed, Aug 10, 2011 at 5:58 PM, Paul Davis <paul.joseph.davis@gmail.com> w=
rote:
>
> Since no one seems to have believed me I decided to take a closer look

I believe you, and in my machine, replication.js, takes about 120ms.

> at replication.js tests. And as I pointed out it was just polling a
> URL in a tight loop for 3s at a time. On my machine, this patch drops
> replication.js from 93329ms to 41785ms

That's awesome. If it doesn't make assertions fails for others, go ahead.

One thing I noticed in the past is that the browser seems to cache the
results of db.info() call. A solution (that is employed somewhere
else, but for another request) is to add some random parameter to the
query string, like  /db?anticache=3DMath.random(1000000).

>. I'll point out that that's
> more than twice as fast. And that was just an obvious optimization
> from watching the log scroll. There are plenty more simple things that
> could be done to speed these up.
>
> Also, this patch makes me think that a _replication/localid -> JSON
> status blob might be useful. Though I dunno how possible that is. I
> reckon if we had that these would be sped up even more.
>
>
> diff --git a/share/www/script/couch.js b/share/www/script/couch.js
> index 304c9c1..792e638 100644
> --- a/share/www/script/couch.js
> +++ b/share/www/script/couch.js
> @@ -40,6 +40,8 @@ function CouchDB(name, httpHeaders) {
> =C2=A0 =C2=A0 if (this.last_req.status =3D=3D 404) {
> =C2=A0 =C2=A0 =C2=A0 return false;
> =C2=A0 =C2=A0 }
> + =C2=A0 =C2=A0var t0 =3D new Date();
> + =C2=A0 =C2=A0while(true) {if((new Date()) - t0 > 100) break;}
> =C2=A0 =C2=A0 CouchDB.maybeThrowError(this.last_req);
> =C2=A0 =C2=A0 return JSON.parse(this.last_req.responseText);
> =C2=A0 };
> diff --git a/share/www/script/test/replication.js
> b/share/www/script/test/replication.js
> index 65c5eaa..b82375a 100644
> --- a/share/www/script/test/replication.js
> +++ b/share/www/script/test/replication.js
> @@ -149,24 +149,40 @@ couchTests.replication =3D function(debug) {
> =C2=A0 }
>
>
> - =C2=A0function waitForSeq(sourceDb, targetDb) {
> - =C2=A0 =C2=A0var targetSeq,
> - =C2=A0 =C2=A0 =C2=A0 =C2=A0sourceSeq =3D sourceDb.info().update_seq,
> + =C2=A0function waitForSeq(sourceDb, targetDb, rep_id) {
> + =C2=A0 =C2=A0var seq =3D sourceDb.info().update_seq,
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0ri =3D new RegExp(rep_id),
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0tasks,
> =C2=A0 =C2=A0 =C2=A0 =C2=A0 t0 =3D new Date(),
> =C2=A0 =C2=A0 =C2=A0 =C2=A0 t1,
> =C2=A0 =C2=A0 =C2=A0 =C2=A0 ms =3D 3000;
>
> =C2=A0 =C2=A0 do {
> - =C2=A0 =C2=A0 =C2=A0targetSeq =3D targetDb.info().update_seq;
> + =C2=A0 =C2=A0 =C2=A0tasks =3D JSON.parse(CouchDB.request("GET",
> "/_active_tasks").responseText);
> + =C2=A0 =C2=A0 =C2=A0for(var i =3D 0; i < tasks.length; i++) {
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0if(!ri.test(tasks[i].task)) continue;
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0var captured =3D /Processed (\d+)/.exec(task=
s[i].status);
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0if(parseInt(captured[1]) >=3D seq) return;
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0break;
> + =C2=A0 =C2=A0 =C2=A0}
> =C2=A0 =C2=A0 =C2=A0 t1 =3D new Date();
> - =C2=A0 =C2=A0} while (((t1 - t0) <=3D ms) && targetSeq < sourceSeq);
> + =C2=A0 =C2=A0} while ((t1 - t0) <=3D ms);
> =C2=A0 }
>
> + =C2=A0function waitForRepEnd(rep_id) {
> + =C2=A0 =C2=A0var ri =3D new RegExp(rep_id),
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0tasks,
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0t0 =3D new Date(),
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0t1,
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0ms =3D 3000;
>
> - =C2=A0function wait(ms) {
> - =C2=A0 =C2=A0var t0 =3D new Date(), t1;
> =C2=A0 =C2=A0 do {
> - =C2=A0 =C2=A0 =C2=A0CouchDB.request("GET", "/");
> + =C2=A0 =C2=A0 =C2=A0tasks =3D JSON.parse(CouchDB.request("GET",
> "/_active_tasks").responseText);
> + =C2=A0 =C2=A0 =C2=A0var found =3D false;
> + =C2=A0 =C2=A0 =C2=A0for(var i =3D 0; i < tasks.length; i++) {
> + =C2=A0 =C2=A0 =C2=A0 =C2=A0if(!ri.test(tasks[i].task)) found =3D true;
> + =C2=A0 =C2=A0 =C2=A0}
> + =C2=A0 =C2=A0 =C2=A0if(!found) return;
> =C2=A0 =C2=A0 =C2=A0 t1 =3D new Date();
> =C2=A0 =C2=A0 } while ((t1 - t0) <=3D ms);
> =C2=A0 }
> @@ -1143,7 +1159,7 @@ couchTests.replication =3D function(debug) {
>
> =C2=A0 =C2=A0 var rep_id =3D repResult._local_id;
>
> - =C2=A0 =C2=A0waitForSeq(sourceDb, targetDb);
> + =C2=A0 =C2=A0waitForSeq(sourceDb, targetDb, rep_id);
>
> =C2=A0 =C2=A0 for (j =3D 0; j < docs.length; j++) {
> =C2=A0 =C2=A0 =C2=A0 doc =3D docs[j];
> @@ -1181,7 +1197,7 @@ couchTests.replication =3D function(debug) {
> =C2=A0 =C2=A0 var ddoc =3D docs[docs.length - 1]; // design doc
> =C2=A0 =C2=A0 addAtt(sourceDb, ddoc, "readme.txt", att1_data, "text/plain=
");
>
> - =C2=A0 =C2=A0waitForSeq(sourceDb, targetDb);
> + =C2=A0 =C2=A0waitForSeq(sourceDb, targetDb, rep_id);
>
> =C2=A0 =C2=A0 var modifDocs =3D docs.slice(10, 15).concat([ddoc]);
> =C2=A0 =C2=A0 for (j =3D 0; j < modifDocs.length; j++) {
> @@ -1226,7 +1242,7 @@ couchTests.replication =3D function(debug) {
> =C2=A0 =C2=A0 // add another attachment to the ddoc on source
> =C2=A0 =C2=A0 addAtt(sourceDb, ddoc, "data.dat", att2_data, "application/=
binary");
>
> - =C2=A0 =C2=A0waitForSeq(sourceDb, targetDb);
> + =C2=A0 =C2=A0waitForSeq(sourceDb, targetDb, rep_id);
>
> =C2=A0 =C2=A0 copy =3D targetDb.open(ddoc._id);
> =C2=A0 =C2=A0 var atts =3D copy._attachments;
> @@ -1263,7 +1279,7 @@ couchTests.replication =3D function(debug) {
> =C2=A0 =C2=A0 var newDocs =3D makeDocs(25, 35);
> =C2=A0 =C2=A0 populateDb(sourceDb, newDocs, true);
>
> - =C2=A0 =C2=A0waitForSeq(sourceDb, targetDb);
> + =C2=A0 =C2=A0waitForSeq(sourceDb, targetDb, rep_id);
>
> =C2=A0 =C2=A0 for (j =3D 0; j < newDocs.length; j++) {
> =C2=A0 =C2=A0 =C2=A0 doc =3D newDocs[j];
> @@ -1282,7 +1298,7 @@ couchTests.replication =3D function(debug) {
> =C2=A0 =C2=A0 TEquals(true, sourceDb.deleteDoc(newDocs[0]).ok);
> =C2=A0 =C2=A0 TEquals(true, sourceDb.deleteDoc(newDocs[6]).ok);
>
> - =C2=A0 =C2=A0waitForSeq(sourceDb, targetDb);
> + =C2=A0 =C2=A0waitForSeq(sourceDb, targetDb, rep_id);
>
> =C2=A0 =C2=A0 copy =3D targetDb.open(newDocs[0]._id);
> =C2=A0 =C2=A0 TEquals(null, copy);
> @@ -1317,7 +1333,7 @@ couchTests.replication =3D function(debug) {
> =C2=A0 =C2=A0 };
> =C2=A0 =C2=A0 TEquals(true, sourceDb.save(doc).ok);
>
> - =C2=A0 =C2=A0wait(2000);
> + =C2=A0 =C2=A0waitForRepEnd(rep_id);
> =C2=A0 =C2=A0 copy =3D targetDb.open(doc._id);
> =C2=A0 =C2=A0 TEquals(null, copy);
> =C2=A0 }
> @@ -1359,7 +1375,7 @@ couchTests.replication =3D function(debug) {
>
> =C2=A0 var tasksAfter =3D JSON.parse(xhr.responseText);
> =C2=A0 TEquals(tasks.length, tasksAfter.length);
> - =C2=A0waitForSeq(sourceDb, targetDb);
> + =C2=A0waitForSeq(sourceDb, targetDb, rep_id);
> =C2=A0 T(sourceDb.open("30") !=3D=3D null);
>
> =C2=A0 // cancel replication
>


--=20
Filipe David Manana,
fdmanana@gmail.com, fdmanana@apache.org

"Reasonable men adapt themselves to the world.
=C2=A0Unreasonable men adapt the world to themselves.
=C2=A0That's why all progress depends on unreasonable men."