hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e...@apache.org
Subject [2/3] hbase git commit: HBASE-14951 Make hbase.regionserver.maxlogs obsolete
Date Wed, 16 Dec 2015 23:06:53 GMT
HBASE-14951 Make hbase.regionserver.maxlogs obsolete


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8bd0d9da
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8bd0d9da
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8bd0d9da

Branch: refs/heads/branch-1
Commit: 8bd0d9dacd5c9d600615f65787323683850afeb2
Parents: 42518fa
Author: Enis Soztutar <enis@apache.org>
Authored: Wed Dec 16 14:57:59 2015 -0800
Committer: Enis Soztutar <enis@apache.org>
Committed: Wed Dec 16 14:58:05 2015 -0800

----------------------------------------------------------------------
 .../hadoop/hbase/regionserver/wal/FSHLog.java   | 21 ++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/8bd0d9da/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
index 6e39517..4329ce5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
@@ -23,6 +23,8 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.io.OutputStream;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryUsage;
 import java.lang.reflect.InvocationTargetException;
 import java.net.URLEncoder;
 import java.util.ArrayList;
@@ -62,6 +64,7 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.apache.hadoop.hbase.util.DrainBarrier;
@@ -505,8 +508,16 @@ public class FSHLog implements WAL {
         FSUtils.getDefaultBlockSize(this.fs, this.fullPathLogDir));
     this.logrollsize =
       (long)(blocksize * conf.getFloat("hbase.regionserver.logroll.multiplier", 0.95f));
-
-    this.maxLogs = conf.getInt("hbase.regionserver.maxlogs", 32);
+    
+    float memstoreRatio = conf.getFloat(HeapMemorySizeUtil.MEMSTORE_SIZE_KEY,
+      conf.getFloat(HeapMemorySizeUtil.MEMSTORE_SIZE_OLD_KEY, 
+        HeapMemorySizeUtil.DEFAULT_MEMSTORE_SIZE));
+    boolean maxLogsDefined = conf.get("hbase.regionserver.maxlogs") != null;
+    if(maxLogsDefined){
+      LOG.warn("'hbase.regionserver.maxlogs' was deprecated.");
+    }
+    this.maxLogs = conf.getInt("hbase.regionserver.maxlogs", 
+        Math.max(32, calculateMaxLogFiles(memstoreRatio, logrollsize)));    
     this.minTolerableReplication = conf.getInt("hbase.regionserver.hlog.tolerable.lowreplication",
         FSUtils.getDefaultReplication(fs, this.fullPathLogDir));
     this.lowReplicationRollLimit =
@@ -556,6 +567,12 @@ public class FSHLog implements WAL {
     this.disruptor.start();
   }
 
+  private int calculateMaxLogFiles(float memstoreSizeRatio, long logRollSize) {
+    MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
+    int maxLogs = Math.round(mu.getMax() * memstoreSizeRatio * 2 / logRollSize);
+    return maxLogs;
+  }
+  
   /**
    * Get the backing files associated with this WAL.
    * @return may be null if there are no files.


Mime
View raw message