hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r797683 - in /hadoop/hbase/trunk: CHANGES.txt src/java/org/apache/hadoop/hbase/HConstants.java src/java/org/apache/hadoop/hbase/regionserver/HLog.java
Date Fri, 24 Jul 2009 22:47:10 GMT
Author: stack
Date: Fri Jul 24 22:47:10 2009
New Revision: 797683

URL: http://svn.apache.org/viewvc?rev=797683&view=rev
Log:
HBASE-1683 OOME on master splitting logs; stuck, won't go down

Modified:
    hadoop/hbase/trunk/CHANGES.txt
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HLog.java

Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=797683&r1=797682&r2=797683&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Fri Jul 24 22:47:10 2009
@@ -510,6 +510,7 @@
                ITHBase
    HBASE-1699  Remove hbrep example as it's too out of date
                (Tim Sell via Stack)
+   HBASE-1683  OOME on master splitting logs; stuck, won't go down
 
   OPTIMIZATIONS
    HBASE-1412  Change values for delete column and column family in KeyValue

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java?rev=797683&r1=797682&r2=797683&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HConstants.java Fri Jul 24 22:47:10
2009
@@ -135,15 +135,7 @@
   
   /** Default size of a reservation block   */
   static final int DEFAULT_SIZE_RESERVATION_BLOCK = 1024 * 1024 * 5;
-  
-  /** Default number of threads to use when log splitting 
-   *  to rewrite the logs. More means faster but bigger mem consumption */
-  static final int DEFAULT_NUMBER_LOG_WRITER_THREAD = 10;
-  
-  /** Default number of logs to read concurrently
-   *  when log splitting. More means faster but bigger mem consumption  */
-  static final int DEFAULT_NUMBER_CONCURRENT_LOG_READS = 10;
-  
+
   /** Maximum value length, enforced on KeyValue construction */
   static final int MAXIMUM_VALUE_LENGTH = Integer.MAX_VALUE;
   

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HLog.java?rev=797683&r1=797682&r2=797683&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HLog.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HLog.java Fri Jul 24
22:47:10 2009
@@ -838,16 +838,26 @@
     final Map<byte [], WriterAndPath> logWriters =
       new TreeMap<byte [], WriterAndPath>(Bytes.BYTES_COMPARATOR);
     List<Path> splits = null;
+    
+    // Number of threads to use when log splitting to rewrite the logs.
+    // More means faster but bigger mem consumption.
+    int logWriterThreads =
+      conf.getInt("hbase.regionserver.hlog.splitlog.writer.threads", 3);
+    
+    // Number of logs to read concurrently when log splitting.
+    // More means faster but bigger mem consumption  */
+    int concurrentLogReads =
+      conf.getInt("hbase.regionserver.hlog.splitlog.reader.threads", 3);
+
     try {
       int maxSteps = Double.valueOf(Math.ceil((logfiles.length * 1.0) / 
-          DEFAULT_NUMBER_CONCURRENT_LOG_READS)).intValue();
+          concurrentLogReads)).intValue();
       for(int step = 0; step < maxSteps; step++) {
         final Map<byte[], LinkedList<HLogEntry>> logEntries = 
           new TreeMap<byte[], LinkedList<HLogEntry>>(Bytes.BYTES_COMPARATOR);
         // Stop at logfiles.length when it's the last step
         int endIndex = step == maxSteps - 1? logfiles.length: 
-          step * DEFAULT_NUMBER_CONCURRENT_LOG_READS +
-          DEFAULT_NUMBER_CONCURRENT_LOG_READS;
+          step * concurrentLogReads + concurrentLogReads;
         for (int i = (step * 10); i < endIndex; i++) {
           // Check for possibly empty file. With appends, currently Hadoop 
           // reports a zero length even if the file has been sync'd. Revisit if
@@ -915,8 +925,8 @@
             fs.delete(logfiles[i].getPath(), true);
           }
         }
-        ExecutorService threadPool = 
-          Executors.newFixedThreadPool(DEFAULT_NUMBER_LOG_WRITER_THREAD);
+        ExecutorService threadPool =
+          Executors.newFixedThreadPool(logWriterThreads);
         for (final byte[] key : logEntries.keySet()) {
           Thread thread = new Thread(Bytes.toString(key)) {
             @Override



Mime
View raw message