hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r723417 - in /hadoop/hbase/trunk: CHANGES.txt conf/hbase-default.xml src/java/org/apache/hadoop/hbase/regionserver/MemcacheFlusher.java src/test/org/apache/hadoop/hbase/TestGlobalMemcacheLimit.java
Date Thu, 04 Dec 2008 20:01:06 GMT
Author: stack
Date: Thu Dec  4 12:01:06 2008
New Revision: 723417

URL: http://svn.apache.org/viewvc?rev=723417&view=rev
Log:
HBASE-1027 Make global flusher check work with percentages rather than hard code memory sizes.

Removed:
    hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/TestGlobalMemcacheLimit.java
Modified:
    hadoop/hbase/trunk/CHANGES.txt
    hadoop/hbase/trunk/conf/hbase-default.xml
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/MemcacheFlusher.java

Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=723417&r1=723416&r2=723417&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Thu Dec  4 12:01:06 2008
@@ -97,6 +97,8 @@
    HBASE-927   We don't recover if HRS hosting -ROOT-/.META. goes down -
                (fix bug in createTable which caused tests to fail)
    HBASE-1039  Compaction fails if bloomfilters are enabled
+   HBASE-1027  Make global flusher check work with percentages rather than
+               hard code memory sizes
 
   IMPROVEMENTS
    HBASE-901   Add a limit to key length, check key and value length on client side

Modified: hadoop/hbase/trunk/conf/hbase-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/conf/hbase-default.xml?rev=723417&r1=723416&r2=723417&view=diff
==============================================================================
--- hadoop/hbase/trunk/conf/hbase-default.xml (original)
+++ hadoop/hbase/trunk/conf/hbase-default.xml Thu Dec  4 12:01:06 2008
@@ -191,19 +191,19 @@
     </description>
   </property>
   <property>
-    <name>hbase.regionserver.globalMemcacheLimit</name>
-    <value>536870912</value>
+    <name>hbase.regionserver.globalMemcache.upperLimit</name>
+    <value>0.4</value>
     <description>Maximum size of all memcaches in a region server before new 
-      updates are blocked and flushes are forced. Defaults to 512MB.
+      updates are blocked and flushes are forced. Defaults to 40% of heap.
     </description>
   </property>
   <property>
-    <name>hbase.regionserver.globalMemcacheLimitlowMark</name>
-    <value>256435456</value>
+    <name>hbase.regionserver.globalMemcache.lowerLimit</name>
+    <value>0.25</value>
     <description>When memcaches are being forced to flush to make room in
-      memory, keep flushing until we hit this mark. Defaults to 256MB. Setting
-      this value equal to hbase.regionserver.globalmemcachelimit causes the 
-      minimum possible flushing to occur when updates are blocked due to 
+      memory, keep flushing until we hit this mark. Defaults to 30% of heap. 
+      This value equal to hbase.regionserver.globalmemcache.upperLimit causes
+      the minimum possible flushing to occur when updates are blocked due to 
       memcache limiting.
     </description>
   </property>  

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/MemcacheFlusher.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/MemcacheFlusher.java?rev=723417&r1=723416&r2=723417&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/MemcacheFlusher.java
(original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/MemcacheFlusher.java
Thu Dec  4 12:01:06 2008
@@ -20,20 +20,22 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.ConcurrentModificationException;
+import java.util.HashSet;
+import java.util.SortedMap;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.locks.ReentrantLock;
 import java.util.concurrent.TimeUnit;
-import java.util.HashSet;
-import java.util.SortedMap;
-import java.util.ConcurrentModificationException;
+import java.util.concurrent.locks.ReentrantLock;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.DroppedSnapshotException;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
+import org.apache.hadoop.util.StringUtils;
 
 /**
  * Thread that flushes cache on request
@@ -58,6 +60,13 @@
   protected final long globalMemcacheLimit;
   protected final long globalMemcacheLimitLowMark;
   
+  public static final float DEFAULT_UPPER = 0.4f;
+  public static final float DEFAULT_LOWER = 0.25f;
+  public static final String UPPER_KEY =
+    "hbase.regionserver.globalMemcache.upperLimit";
+  public static final String LOWER_KEY =
+    "hbase.regionserver.globalMemcache.lowerLimit";
+  
   /**
    * @param conf
    * @param server
@@ -66,16 +75,47 @@
       final HRegionServer server) {
     super();
     this.server = server;
-    threadWakeFrequency = conf.getLong(
-        HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000);
-        
-    // default memcache limit of 512MB
-    globalMemcacheLimit = 
-      conf.getLong("hbase.regionserver.globalMemcacheLimit", 512 * 1024 * 1024);
-    // default memcache low mark limit of 256MB, which is half the upper limit
-    globalMemcacheLimitLowMark = 
-      conf.getLong("hbase.regionserver.globalMemcacheLimitLowMark", 
-        globalMemcacheLimit / 2);        
+    this.threadWakeFrequency =
+      conf.getLong(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000);
+    long max = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
+    this.globalMemcacheLimit = globalMemcacheLimit(max, DEFAULT_UPPER,
+      UPPER_KEY, conf);
+    long lower = globalMemcacheLimit(max, DEFAULT_LOWER, LOWER_KEY, conf);
+    if (lower > this.globalMemcacheLimit) {
+      lower = this.globalMemcacheLimit;
+      LOG.info("Setting globalMemcacheLimitLowMark == globalMemcacheLimit " +
+        "because supplied " + LOWER_KEY + " was > " + UPPER_KEY);
+    }
+    this.globalMemcacheLimitLowMark = lower;
+    LOG.info("globalMemcacheLimit=" +
+      StringUtils.humanReadableInt(this.globalMemcacheLimit) +
+      ", globalMemcacheLimitLowMark=" +
+      StringUtils.humanReadableInt(this.globalMemcacheLimitLowMark) +
+      ", maxHeap=" + StringUtils.humanReadableInt(max));
+  }
+
+  /**
+   * Calculate size using passed <code>key</code> for configured
+   * percentage of <code>max</code>.
+   * @param max
+   * @param defaultLimit
+   * @param key
+   * @param c
+   * @return Limit.
+   */
+  static long globalMemcacheLimit(final long max,
+     final float defaultLimit, final String key, final HBaseConfiguration c) {
+    float limit = c.getFloat(key, defaultLimit);
+    return getMemcacheLimit(max, limit, defaultLimit);
+  }
+  
+  static long getMemcacheLimit(final long max, final float limit,
+      final float defaultLimit) {
+    if (limit >= 0.9f || limit < 0.1f) {
+      LOG.warn("Setting global memcache limit to default of " + defaultLimit +
+        " because supplied value outside allowed range of 0.1 -> 0.9");
+    }
+    return (long)(max * limit);
   }
   
   @Override
@@ -229,15 +269,18 @@
       // flush the region with the biggest memcache
       if (m.size() <= 0) {
         LOG.info("No online regions to flush though we've been asked flush " +
-            "some; globalMemcacheSize=" + globalMemcacheSize +
-            ", globalMemcacheLimitLowMark=" + this.globalMemcacheLimitLowMark);
+          "some; globalMemcacheSize=" +
+          StringUtils.humanReadableInt(globalMemcacheSize) +
+          ", globalMemcacheLimitLowMark=" +
+          StringUtils.humanReadableInt(this.globalMemcacheLimitLowMark));
         break;
       }
       HRegion biggestMemcacheRegion = m.remove(m.firstKey());
       LOG.info("Forced flushing of " +  biggestMemcacheRegion.toString() +
         " because global memcache limit of " + this.globalMemcacheLimit +
-        " exceeded; currently " + globalMemcacheSize + " and flushing till " +
-        this.globalMemcacheLimitLowMark);
+        " exceeded; currently " +
+        StringUtils.humanReadableInt(globalMemcacheSize) + " and flushing till " +
+        StringUtils.humanReadableInt(this.globalMemcacheLimitLowMark));
       if (!flushRegion(biggestMemcacheRegion, true)) {
         LOG.warn("Flush failed");
         break;



Mime
View raw message