hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From syuanji...@apache.org
Subject [12/12] hbase git commit: HBASE-14951 Make hbase.regionserver.maxlogs obsolete
Date Wed, 16 Dec 2015 23:45:23 GMT
HBASE-14951 Make hbase.regionserver.maxlogs obsolete


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3dec8582
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3dec8582
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3dec8582

Branch: refs/heads/hbase-12439
Commit: 3dec8582f527e4e7e35280cd34c4928648290658
Parents: e965b61
Author: Enis Soztutar <enis@apache.org>
Authored: Wed Dec 16 14:57:59 2015 -0800
Committer: Enis Soztutar <enis@apache.org>
Committed: Wed Dec 16 14:57:59 2015 -0800

----------------------------------------------------------------------
 .../hadoop/hbase/regionserver/wal/FSHLog.java   | 21 ++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/3dec8582/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
index 0d1ff0d..9ae72e6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
@@ -23,6 +23,8 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.io.OutputStream;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryUsage;
 import java.lang.reflect.InvocationTargetException;
 import java.net.URLEncoder;
 import java.util.ArrayList;
@@ -62,6 +64,7 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.apache.hadoop.hbase.util.DrainBarrier;
@@ -506,8 +509,16 @@ public class FSHLog implements WAL {
         FSUtils.getDefaultBlockSize(this.fs, this.fullPathLogDir));
     this.logrollsize =
       (long)(blocksize * conf.getFloat("hbase.regionserver.logroll.multiplier", 0.95f));
-
-    this.maxLogs = conf.getInt("hbase.regionserver.maxlogs", 32);
+    
+    float memstoreRatio = conf.getFloat(HeapMemorySizeUtil.MEMSTORE_SIZE_KEY,
+      conf.getFloat(HeapMemorySizeUtil.MEMSTORE_SIZE_OLD_KEY, 
+        HeapMemorySizeUtil.DEFAULT_MEMSTORE_SIZE));
+    boolean maxLogsDefined = conf.get("hbase.regionserver.maxlogs") != null;
+    if(maxLogsDefined){
+      LOG.warn("'hbase.regionserver.maxlogs' was deprecated.");
+    }
+    this.maxLogs = conf.getInt("hbase.regionserver.maxlogs", 
+        Math.max(32, calculateMaxLogFiles(memstoreRatio, logrollsize)));    
     this.minTolerableReplication = conf.getInt("hbase.regionserver.hlog.tolerable.lowreplication",
         FSUtils.getDefaultReplication(fs, this.fullPathLogDir));
     this.lowReplicationRollLimit =
@@ -557,6 +568,12 @@ public class FSHLog implements WAL {
     this.disruptor.start();
   }
 
+  private int calculateMaxLogFiles(float memstoreSizeRatio, long logRollSize) {
+    MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
+    int maxLogs = Math.round(mu.getMax() * memstoreSizeRatio * 2 / logRollSize);
+    return maxLogs;
+  }
+  
   /**
    * Get the backing files associated with this WAL.
    * @return may be null if there are no files.


Mime
View raw message