From commits-return-5274-apmail-couchdb-commits-archive=couchdb.apache.org@couchdb.apache.org Tue Oct 05 18:25:38 2010 Return-Path: Delivered-To: apmail-couchdb-commits-archive@www.apache.org Received: (qmail 71345 invoked from network); 5 Oct 2010 18:25:37 -0000 Received: from unknown (HELO mail.apache.org) (140.211.11.3) by 140.211.11.9 with SMTP; 5 Oct 2010 18:25:37 -0000 Received: (qmail 85852 invoked by uid 500); 5 Oct 2010 18:25:37 -0000 Delivered-To: apmail-couchdb-commits-archive@couchdb.apache.org Received: (qmail 85793 invoked by uid 500); 5 Oct 2010 18:25:37 -0000 Mailing-List: contact commits-help@couchdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@couchdb.apache.org Delivered-To: mailing list commits@couchdb.apache.org Received: (qmail 85784 invoked by uid 99); 5 Oct 2010 18:25:37 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 05 Oct 2010 18:25:37 +0000 X-ASF-Spam-Status: No, hits=-1996.4 required=10.0 tests=ALL_TRUSTED,FS_REPLICA X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 05 Oct 2010 18:25:29 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 5C34223889E3; Tue, 5 Oct 2010 18:25:08 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1004758 - in /couchdb/branches/new_replicator: etc/couchdb/default.ini.tpl.in src/couchdb/couch_replicator.erl src/couchdb/couch_replicator_doc_copiers.erl src/couchdb/couch_replicator_rev_finders.erl Date: Tue, 05 Oct 2010 18:25:08 -0000 To: commits@couchdb.apache.org From: fdmanana@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20101005182508.5C34223889E3@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: fdmanana Date: Tue Oct 5 18:25:07 2010 New Revision: 1004758 URL: http://svn.apache.org/viewvc?rev=1004758&view=rev Log: New replicator: add better default values and .ini option "buffer_size". Modified: couchdb/branches/new_replicator/etc/couchdb/default.ini.tpl.in couchdb/branches/new_replicator/src/couchdb/couch_replicator.erl couchdb/branches/new_replicator/src/couchdb/couch_replicator_doc_copiers.erl couchdb/branches/new_replicator/src/couchdb/couch_replicator_rev_finders.erl Modified: couchdb/branches/new_replicator/etc/couchdb/default.ini.tpl.in URL: http://svn.apache.org/viewvc/couchdb/branches/new_replicator/etc/couchdb/default.ini.tpl.in?rev=1004758&r1=1004757&r2=1004758&view=diff ============================================================================== --- couchdb/branches/new_replicator/etc/couchdb/default.ini.tpl.in (original) +++ couchdb/branches/new_replicator/etc/couchdb/default.ini.tpl.in Tue Oct 5 18:25:07 2010 @@ -118,3 +118,5 @@ compressible_types = text/*, application [replicator] ; should be at least 2 worker_processes = 10 +; integer value in bytes +buffer_size = 1048576 \ No newline at end of file Modified: couchdb/branches/new_replicator/src/couchdb/couch_replicator.erl URL: http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_replicator.erl?rev=1004758&r1=1004757&r2=1004758&view=diff ============================================================================== --- couchdb/branches/new_replicator/src/couchdb/couch_replicator.erl (original) +++ couchdb/branches/new_replicator/src/couchdb/couch_replicator.erl Tue Oct 5 18:25:07 2010 @@ -217,8 +217,10 @@ do_init(#rep{options = Options} = Rep) - start_seq = StartSeq } = State = init_state(Rep), + QueueSize = round( + (?l2i(couch_config:get("replicator", "buffer_size", "1048576"))) / 2), {ok, MissingRevsQueue} = couch_work_queue:new( - [{max_size, 100000}, {max_items, 2000}, {multi_workers, true}]), + [{max_size, QueueSize}, {multi_workers, true}]), {RevFindersCount, CopiersCount} = case ?l2i( couch_config:get("replicator", "worker_processes", "10")) of @@ -227,13 +229,13 @@ do_init(#rep{options = Options} = Rep) - "should be at least 2", []), {1, 1}; N -> - {N div 2, (N div 2) + (N rem 2)} + {lists:max([N div 3, 1]), lists:max([2 * (N div 3) + (N rem 3), 1])} end, case get_value(doc_ids, Options) of undefined -> {ok, ChangesQueue} = couch_work_queue:new( - [{max_size, 100000}, {max_items, 500}, {multi_workers, true}]), + [{max_size, QueueSize}, {multi_workers, true}]), % This starts the _changes reader process. It adds the changes from % the source db to the ChangesQueue. Modified: couchdb/branches/new_replicator/src/couchdb/couch_replicator_doc_copiers.erl URL: http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_replicator_doc_copiers.erl?rev=1004758&r1=1004757&r2=1004758&view=diff ============================================================================== --- couchdb/branches/new_replicator/src/couchdb/couch_replicator_doc_copiers.erl (original) +++ couchdb/branches/new_replicator/src/couchdb/couch_replicator_doc_copiers.erl Tue Oct 5 18:25:07 2010 @@ -17,7 +17,7 @@ -include("couch_db.hrl"). -include("couch_api_wrap.hrl"). --define(DOC_BATCH_SIZE, 50). +-define(DOC_BATCH_SIZE, 1000). Modified: couchdb/branches/new_replicator/src/couchdb/couch_replicator_rev_finders.erl URL: http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_replicator_rev_finders.erl?rev=1004758&r1=1004757&r2=1004758&view=diff ============================================================================== --- couchdb/branches/new_replicator/src/couchdb/couch_replicator_rev_finders.erl (original) +++ couchdb/branches/new_replicator/src/couchdb/couch_replicator_rev_finders.erl Tue Oct 5 18:25:07 2010 @@ -16,7 +16,7 @@ -include("couch_db.hrl"). --define(REV_BATCH_SIZE, 100). +-define(REV_BATCH_SIZE, 3000).