Return-Path: Delivered-To: apmail-hadoop-hbase-commits-archive@minotaur.apache.org Received: (qmail 54339 invoked from network); 24 Jul 2009 22:46:27 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 24 Jul 2009 22:46:27 -0000 Received: (qmail 63954 invoked by uid 500); 24 Jul 2009 22:47:32 -0000 Delivered-To: apmail-hadoop-hbase-commits-archive@hadoop.apache.org Received: (qmail 63910 invoked by uid 500); 24 Jul 2009 22:47:32 -0000 Mailing-List: contact hbase-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hbase-dev@hadoop.apache.org Delivered-To: mailing list hbase-commits@hadoop.apache.org Received: (qmail 63901 invoked by uid 99); 24 Jul 2009 22:47:32 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 24 Jul 2009 22:47:32 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 24 Jul 2009 22:47:30 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 8C6432388877; Fri, 24 Jul 2009 22:47:10 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r797683 - in /hadoop/hbase/trunk: CHANGES.txt src/java/org/apache/hadoop/hbase/HConstants.java src/java/org/apache/hadoop/hbase/regionserver/HLog.java Date: Fri, 24 Jul 2009 22:47:10 -0000 To: hbase-commits@hadoop.apache.org From: stack@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20090724224710.8C6432388877@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: stack Date: Fri Jul 24 22:47:10 2009 New Revision: 797683 URL: http://svn.apache.org/viewvc?rev=797683&view=rev Log: HBASE-1683 OOME on master splitting logs; stuck, won't go down Modified: hadoop/hbase/trunk/CHANGES.txt hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HLog.java Modified: hadoop/hbase/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=797683&r1=797682&r2=797683&view=diff ============================================================================== --- hadoop/hbase/trunk/CHANGES.txt (original) +++ hadoop/hbase/trunk/CHANGES.txt Fri Jul 24 22:47:10 2009 @@ -510,6 +510,7 @@ ITHBase HBASE-1699 Remove hbrep example as it's too out of date (Tim Sell via Stack) + HBASE-1683 OOME on master splitting logs; stuck, won't go down OPTIMIZATIONS HBASE-1412 Change values for delete column and column family in KeyValue Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java?rev=797683&r1=797682&r2=797683&view=diff ============================================================================== --- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java (original) +++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java Fri Jul 24 22:47:10 2009 @@ -135,15 +135,7 @@ /** Default size of a reservation block */ static final int DEFAULT_SIZE_RESERVATION_BLOCK = 1024 * 1024 * 5; - - /** Default number of threads to use when log splitting - * to rewrite the logs. More means faster but bigger mem consumption */ - static final int DEFAULT_NUMBER_LOG_WRITER_THREAD = 10; - - /** Default number of logs to read concurrently - * when log splitting. More means faster but bigger mem consumption */ - static final int DEFAULT_NUMBER_CONCURRENT_LOG_READS = 10; - + /** Maximum value length, enforced on KeyValue construction */ static final int MAXIMUM_VALUE_LENGTH = Integer.MAX_VALUE; Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HLog.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HLog.java?rev=797683&r1=797682&r2=797683&view=diff ============================================================================== --- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HLog.java (original) +++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HLog.java Fri Jul 24 22:47:10 2009 @@ -838,16 +838,26 @@ final Map logWriters = new TreeMap(Bytes.BYTES_COMPARATOR); List splits = null; + + // Number of threads to use when log splitting to rewrite the logs. + // More means faster but bigger mem consumption. + int logWriterThreads = + conf.getInt("hbase.regionserver.hlog.splitlog.writer.threads", 3); + + // Number of logs to read concurrently when log splitting. + // More means faster but bigger mem consumption */ + int concurrentLogReads = + conf.getInt("hbase.regionserver.hlog.splitlog.reader.threads", 3); + try { int maxSteps = Double.valueOf(Math.ceil((logfiles.length * 1.0) / - DEFAULT_NUMBER_CONCURRENT_LOG_READS)).intValue(); + concurrentLogReads)).intValue(); for(int step = 0; step < maxSteps; step++) { final Map> logEntries = new TreeMap>(Bytes.BYTES_COMPARATOR); // Stop at logfiles.length when it's the last step int endIndex = step == maxSteps - 1? logfiles.length: - step * DEFAULT_NUMBER_CONCURRENT_LOG_READS + - DEFAULT_NUMBER_CONCURRENT_LOG_READS; + step * concurrentLogReads + concurrentLogReads; for (int i = (step * 10); i < endIndex; i++) { // Check for possibly empty file. With appends, currently Hadoop // reports a zero length even if the file has been sync'd. Revisit if @@ -915,8 +925,8 @@ fs.delete(logfiles[i].getPath(), true); } } - ExecutorService threadPool = - Executors.newFixedThreadPool(DEFAULT_NUMBER_LOG_WRITER_THREAD); + ExecutorService threadPool = + Executors.newFixedThreadPool(logWriterThreads); for (final byte[] key : logEntries.keySet()) { Thread thread = new Thread(Bytes.toString(key)) { @Override