hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ecl...@apache.org
Subject hbase git commit: HBASE-15102 Fix HeapMemoryTuner overtuning memstore
Date Tue, 19 Jan 2016 20:24:32 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1 7d7a2d871 -> a61f3ecc8


HBASE-15102 Fix HeapMemoryTuner overtuning memstore

HeapMemoryTuner often over tunes memstore without looking at
the lower limit of the previous memstore size and causing a
situation in which memstore used size suddenly exceeds the
total memstore size.

Signed-off-by: Elliott Clark <eclark@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a61f3ecc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a61f3ecc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a61f3ecc

Branch: refs/heads/branch-1
Commit: a61f3ecc8b3dad256b7ec5258e1ec4aa811cb606
Parents: 7d7a2d8
Author: Ashu Pachauri <ashu210890@gmail.com>
Authored: Wed Jan 13 13:49:43 2016 -0800
Committer: Elliott Clark <eclark@apache.org>
Committed: Tue Jan 19 12:23:59 2016 -0800

----------------------------------------------------------------------
 .../regionserver/DefaultHeapMemoryTuner.java    | 218 +++++++++++--------
 .../regionserver/TestHeapMemoryManager.java     |  23 +-
 2 files changed, 143 insertions(+), 98 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/a61f3ecc/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java
index b6e81dd..1d237d0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java
@@ -124,30 +124,113 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner {
 
   @Override
   public TunerResult tune(TunerContext context) {
-    long blockedFlushCount = context.getBlockedFlushCount();
-    long unblockedFlushCount = context.getUnblockedFlushCount();
-    long evictCount = context.getEvictCount();
-    long cacheMissCount = context.getCacheMissCount();
-    long totalFlushCount = blockedFlushCount+unblockedFlushCount;
-    rollingStatsForCacheMisses.insertDataValue(cacheMissCount);
-    rollingStatsForFlushes.insertDataValue(totalFlushCount);
-    rollingStatsForEvictions.insertDataValue(evictCount);
-    StepDirection newTuneDirection = StepDirection.NEUTRAL;
+    float curMemstoreSize = context.getCurMemStoreSize();
+    float curBlockCacheSize = context.getCurBlockCacheSize();
+    addToRollingStats(context);
+
     if (ignoreInitialPeriods < numPeriodsToIgnore) {
       // Ignoring the first few tuner periods
       ignoreInitialPeriods++;
       rollingStatsForTunerSteps.insertDataValue(0);
       return NO_OP_TUNER_RESULT;
     }
-    String tunerLog = "";
+    StepDirection newTuneDirection = getTuneDirection(context);
+
+    float newMemstoreSize;
+    float newBlockCacheSize;
+
+    // Adjusting step size for tuning to get to steady state or restart from steady state.
+    // Even if the step size was 4% and 32 GB memory size, we will be shifting 1 GB back
and forth
+    // per tuner operation and it can affect the performance of cluster so we keep on decreasing
+    // step size until everything settles.
+    if (prevTuneDirection == StepDirection.NEUTRAL
+        && newTuneDirection != StepDirection.NEUTRAL
+        && rollingStatsForTunerSteps.getDeviation() < TUNER_STEP_EPS) {
+      // Restarting the tuning from steady state and setting step size to maximum.
+      // The deviation cannot be that low if last period was neutral and some recent periods
were
+      // not neutral.
+      step = maximumStepSize;
+    } else if ((newTuneDirection == StepDirection.INCREASE_MEMSTORE_SIZE
+        && decayingTunerStepSizeSum < 0) ||
+        (newTuneDirection == StepDirection.INCREASE_BLOCK_CACHE_SIZE
+        && decayingTunerStepSizeSum > 0)) {
+      // Current step is opposite of past tuner actions so decrease the step size to reach
steady
+      // state.
+      step = step/2.00f;
+    }
+    if (step < minimumStepSize) {
+      // If step size is too small then we do nothing.
+      LOG.debug("Tuner step size is too low; we will not perform any tuning this time.");
+      step = 0.0f;
+      newTuneDirection = StepDirection.NEUTRAL;
+    }
+    // Increase / decrease the memstore / block cahce sizes depending on new tuner step.
+    float globalMemstoreLowerMark = HeapMemorySizeUtil.getGlobalMemStoreLowerMark(conf,
+        curMemstoreSize);
+    // We don't want to exert immediate pressure on memstore. So, we decrease its size gracefully;
+    // we set a minimum bar in the middle of the total memstore size and the lower limit.
+    float minMemstoreSize = ((globalMemstoreLowerMark + 1) * curMemstoreSize) / 2.00f;
+
+    switch (newTuneDirection) {
+    case INCREASE_BLOCK_CACHE_SIZE:
+        if (curMemstoreSize - step < minMemstoreSize) {
+          step = curMemstoreSize - minMemstoreSize;
+        }
+        newMemstoreSize = curMemstoreSize - step;
+        newBlockCacheSize = curBlockCacheSize + step;
+        rollingStatsForTunerSteps.insertDataValue(-(int)(step*100000));
+        decayingTunerStepSizeSum = (decayingTunerStepSizeSum - step)/2.00f;
+        break;
+    case INCREASE_MEMSTORE_SIZE:
+        newBlockCacheSize = curBlockCacheSize - step;
+        newMemstoreSize = curMemstoreSize + step;
+        rollingStatsForTunerSteps.insertDataValue((int)(step*100000));
+        decayingTunerStepSizeSum = (decayingTunerStepSizeSum + step)/2.00f;
+        break;
+    default:
+        prevTuneDirection = StepDirection.NEUTRAL;
+        rollingStatsForTunerSteps.insertDataValue(0);
+        decayingTunerStepSizeSum = (decayingTunerStepSizeSum)/2.00f;
+        return NO_OP_TUNER_RESULT;
+    }
+    // Check we are within max/min bounds.
+    if (newMemstoreSize > globalMemStorePercentMaxRange) {
+      newMemstoreSize = globalMemStorePercentMaxRange;
+    } else if (newMemstoreSize < globalMemStorePercentMinRange) {
+      newMemstoreSize = globalMemStorePercentMinRange;
+    }
+    if (newBlockCacheSize > blockCachePercentMaxRange) {
+      newBlockCacheSize = blockCachePercentMaxRange;
+    } else if (newBlockCacheSize < blockCachePercentMinRange) {
+      newBlockCacheSize = blockCachePercentMinRange;
+    }
+    TUNER_RESULT.setBlockCacheSize(newBlockCacheSize);
+    TUNER_RESULT.setMemstoreSize(newMemstoreSize);
+    prevTuneDirection = newTuneDirection;
+    return TUNER_RESULT;
+  }
+
+  /**
+   * Determine best direction of tuning base on given context.
+   * @param context The tuner context.
+   * @return tuning direction.
+   */
+  private StepDirection getTuneDirection(TunerContext context) {
+    StepDirection newTuneDirection = StepDirection.NEUTRAL;
+    long blockedFlushCount = context.getBlockedFlushCount();
+    long unblockedFlushCount = context.getUnblockedFlushCount();
+    long evictCount = context.getEvictCount();
+    long cacheMissCount = context.getCacheMissCount();
+    long totalFlushCount = blockedFlushCount+unblockedFlushCount;
+    float curMemstoreSize = context.getCurMemStoreSize();
+    float curBlockCacheSize = context.getCurBlockCacheSize();
+    StringBuilder tunerLog = new StringBuilder();
     // We can consider memstore or block cache to be sufficient if
     // we are using only a minor fraction of what have been already provided to it.
     boolean earlyMemstoreSufficientCheck = totalFlushCount == 0
-            || context.getCurMemStoreUsed() < context.getCurMemStoreSize()*sufficientMemoryLevel;
+        || context.getCurMemStoreUsed() < curMemstoreSize * sufficientMemoryLevel;
     boolean earlyBlockCacheSufficientCheck = evictCount == 0 ||
-            context.getCurBlockCacheUsed() < context.getCurBlockCacheSize()*sufficientMemoryLevel;
-    float newMemstoreSize;
-    float newBlockCacheSize;
+        context.getCurBlockCacheUsed() < curBlockCacheSize * sufficientMemoryLevel;
     if (earlyMemstoreSufficientCheck && earlyBlockCacheSufficientCheck) {
       // Both memstore and block cache memory seems to be sufficient. No operation required.
       newTuneDirection = StepDirection.NEUTRAL;
@@ -168,15 +251,15 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner {
       case INCREASE_BLOCK_CACHE_SIZE:
         if ((double)evictCount > rollingStatsForEvictions.getMean() ||
             (double)totalFlushCount > rollingStatsForFlushes.getMean() +
-            rollingStatsForFlushes.getDeviation()/2.00) {
+                rollingStatsForFlushes.getDeviation()/2.00) {
           // Reverting previous step as it was not useful.
           // Tuning failed to decrease evictions or tuning resulted in large number of flushes.
           newTuneDirection = StepDirection.INCREASE_MEMSTORE_SIZE;
-          tunerLog += "Reverting previous tuning.";
+          tunerLog.append("We will revert previous tuning");
           if ((double)evictCount > rollingStatsForEvictions.getMean()) {
-            tunerLog += " As could not decrease evctions sufficiently.";
+            tunerLog.append(" because we could not decrease evictions sufficiently.");
           } else {
-            tunerLog += " As number of flushes rose significantly.";
+            tunerLog.append(" because the number of flushes rose significantly.");
           }
           isReverting = true;
         }
@@ -184,15 +267,15 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner {
       case INCREASE_MEMSTORE_SIZE:
         if ((double)totalFlushCount > rollingStatsForFlushes.getMean() ||
             (double)evictCount > rollingStatsForEvictions.getMean() +
-            rollingStatsForEvictions.getDeviation()/2.00) {
+                rollingStatsForEvictions.getDeviation()/2.00) {
           // Reverting previous step as it was not useful.
           // Tuning failed to decrease flushes or tuning resulted in large number of evictions.
           newTuneDirection = StepDirection.INCREASE_BLOCK_CACHE_SIZE;
-          tunerLog += "Reverting previous tuning.";
+          tunerLog.append("We will revert previous tuning");
           if ((double)totalFlushCount > rollingStatsForFlushes.getMean()) {
-            tunerLog += " As could not decrease flushes sufficiently.";
+            tunerLog.append(" because we could not decrease flushes sufficiently.");
           } else {
-            tunerLog += " As number of evictions rose significantly.";
+            tunerLog.append(" because number of evictions rose significantly.");
           }
           isReverting = true;
         }
@@ -215,97 +298,52 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner {
         if ((double)cacheMissCount < rollingStatsForCacheMisses.getMean() -
             rollingStatsForCacheMisses.getDeviation()*0.80 &&
             (double)totalFlushCount < rollingStatsForFlushes.getMean() -
-            rollingStatsForFlushes.getDeviation()*0.80) {
+                rollingStatsForFlushes.getDeviation()*0.80) {
           // Everything is fine no tuning required
           newTuneDirection = StepDirection.NEUTRAL;
         } else if ((double)cacheMissCount > rollingStatsForCacheMisses.getMean() +
             rollingStatsForCacheMisses.getDeviation()*0.80 &&
             (double)totalFlushCount < rollingStatsForFlushes.getMean() -
-            rollingStatsForFlushes.getDeviation()*0.80) {
+                rollingStatsForFlushes.getDeviation()*0.80) {
           // more misses , increasing cache size
           newTuneDirection = StepDirection.INCREASE_BLOCK_CACHE_SIZE;
-          tunerLog +=
-              "Increasing block cache size as observed increase in number of cache misses.";
+          tunerLog.append(
+              "Going to increase block cache size due to increase in number of cache misses.");
         } else if ((double)cacheMissCount < rollingStatsForCacheMisses.getMean() -
             rollingStatsForCacheMisses.getDeviation()*0.80 &&
             (double)totalFlushCount > rollingStatsForFlushes.getMean() +
-            rollingStatsForFlushes.getDeviation()*0.80) {
+                rollingStatsForFlushes.getDeviation()*0.80) {
           // more flushes , increasing memstore size
           newTuneDirection = StepDirection.INCREASE_MEMSTORE_SIZE;
-          tunerLog += "Increasing memstore size as observed increase in number of flushes.";
+          tunerLog.append("Going to increase memstore size due to increase in number of flushes.");
         } else if (blockedFlushCount > 0 && prevTuneDirection == StepDirection.NEUTRAL)
{
           // we do not want blocked flushes
           newTuneDirection = StepDirection.INCREASE_MEMSTORE_SIZE;
-          tunerLog += "Increasing memstore size as observed "
-                      + blockedFlushCount + " blocked flushes.";
+          tunerLog.append("Going to increase memstore size due to"
+              + blockedFlushCount + " blocked flushes.");
         } else {
           // Default. Not enough facts to do tuning.
+          tunerLog.append("Going to do nothing because we "
+              + "could not determine best tuning direction");
           newTuneDirection = StepDirection.NEUTRAL;
         }
       }
     }
-    // Adjusting step size for tuning to get to steady state or restart from steady state.
-    // Even if the step size was 4% and 32 GB memory size, we will be shifting 1 GB back
and forth
-    // per tuner operation and it can affect the performance of cluster so we keep on decreasing
-    // step size until everything settles.
-    if (prevTuneDirection == StepDirection.NEUTRAL
-        && newTuneDirection != StepDirection.NEUTRAL
-        && rollingStatsForTunerSteps.getDeviation() < TUNER_STEP_EPS) {
-      // Restarting the tuning from steady state and setting step size to maximum.
-      // The deviation cannot be that low if last period was neutral and some recent periods
were
-      // not neutral.
-      step = maximumStepSize;
-    } else if ((newTuneDirection == StepDirection.INCREASE_MEMSTORE_SIZE
-        && decayingTunerStepSizeSum < 0) ||
-        (newTuneDirection == StepDirection.INCREASE_BLOCK_CACHE_SIZE
-        && decayingTunerStepSizeSum > 0)) {
-      // Current step is opposite of past tuner actions so decrease the step size to reach
steady
-      // state.
-      step = step/2.00f;
-    }
-    if (step < minimumStepSize) {
-      // If step size is too small then we do nothing.
-      step = 0.0f;
-      newTuneDirection = StepDirection.NEUTRAL;
-    }
-    // Increase / decrease the memstore / block cahce sizes depending on new tuner step.
-    switch (newTuneDirection) {
-    case INCREASE_BLOCK_CACHE_SIZE:
-        newBlockCacheSize = context.getCurBlockCacheSize() + step;
-        newMemstoreSize = context.getCurMemStoreSize() - step;
-        rollingStatsForTunerSteps.insertDataValue(-(int)(step*100000));
-        decayingTunerStepSizeSum = (decayingTunerStepSizeSum - step)/2.00f;
-        break;
-    case INCREASE_MEMSTORE_SIZE:
-        newBlockCacheSize = context.getCurBlockCacheSize() - step;
-        newMemstoreSize = context.getCurMemStoreSize() + step;
-        rollingStatsForTunerSteps.insertDataValue((int)(step*100000));
-        decayingTunerStepSizeSum = (decayingTunerStepSizeSum + step)/2.00f;
-        break;
-    default:
-        prevTuneDirection = StepDirection.NEUTRAL;
-        rollingStatsForTunerSteps.insertDataValue(0);
-        decayingTunerStepSizeSum = (decayingTunerStepSizeSum)/2.00f;
-        return NO_OP_TUNER_RESULT;
-    }
-    // Check we are within max/min bounds.
-    if (newMemstoreSize > globalMemStorePercentMaxRange) {
-      newMemstoreSize = globalMemStorePercentMaxRange;
-    } else if (newMemstoreSize < globalMemStorePercentMinRange) {
-      newMemstoreSize = globalMemStorePercentMinRange;
-    }
-    if (newBlockCacheSize > blockCachePercentMaxRange) {
-      newBlockCacheSize = blockCachePercentMaxRange;
-    } else if (newBlockCacheSize < blockCachePercentMinRange) {
-      newBlockCacheSize = blockCachePercentMinRange;
-    }
-    TUNER_RESULT.setBlockCacheSize(newBlockCacheSize);
-    TUNER_RESULT.setMemstoreSize(newMemstoreSize);
     if (LOG.isDebugEnabled()) {
-      LOG.debug(tunerLog);
+      LOG.debug(tunerLog.toString());
     }
-    prevTuneDirection = newTuneDirection;
-    return TUNER_RESULT;
+    return newTuneDirection;
+  }
+
+  /**
+   * Add the given context to the rolling tuner stats.
+   * @param context The tuner context.
+   */
+  private void addToRollingStats(TunerContext context) {
+    rollingStatsForCacheMisses.insertDataValue(context.getCacheMissCount());
+    rollingStatsForFlushes.insertDataValue(context.getBlockedFlushCount() +
+        context.getUnblockedFlushCount());
+    rollingStatsForEvictions.insertDataValue(context.getEvictCount());
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hbase/blob/a61f3ecc/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
index 99acf0f..0e72d0d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
@@ -227,6 +227,7 @@ public class TestHeapMemoryManager {
     blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
     regionServerAccounting.setTestMemstoreSize(0);
     Configuration conf = HBaseConfiguration.create();
+    conf.setFloat(HeapMemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f);
     conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
     conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
     conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
@@ -238,6 +239,11 @@ public class TestHeapMemoryManager {
         new RegionServerStub(conf), new RegionServerAccountingStub());
     long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
     long oldBlockCacheSize = blockCache.maxSize;
+    long oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10;
+    long maxTuneSize = oldMemstoreHeapSize -  (oldMemstoreLowerMarkSize + oldMemstoreHeapSize)
/ 2;
+    float maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize;
+    maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE ?
+        DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE:maxStepValue;
     final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
     heapMemoryManager.start(choreService);
     blockCache.evictBlock(null);
@@ -245,20 +251,21 @@ public class TestHeapMemoryManager {
     blockCache.evictBlock(null);
     // Allow the tuner to run once and do necessary memory up
     waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
-    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldMemstoreHeapSize,
-        memStoreFlusher.memstoreSize);
-    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldBlockCacheSize,
-        blockCache.maxSize);
+    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
+    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
     oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
     oldBlockCacheSize = blockCache.maxSize;
+    oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10;
+    maxTuneSize = oldMemstoreHeapSize -  (oldMemstoreLowerMarkSize + oldMemstoreHeapSize)
/ 2;
+    maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize;
+    maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE ?
+        DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE:maxStepValue;
     // Do some more evictions before the next run of HeapMemoryTuner
     blockCache.evictBlock(null);
     // Allow the tuner to run once and do necessary memory up
     waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
-    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldMemstoreHeapSize,
-        memStoreFlusher.memstoreSize);
-    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldBlockCacheSize,
-        blockCache.maxSize);
+    assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
+    assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
   }
 
   @Test


Mime
View raw message