hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e...@apache.org
Subject hbase git commit: HBASE-14387 Compaction improvements: Maximum off-peak compaction size (Vladimir Rodionov)
Date Tue, 10 Nov 2015 19:29:41 GMT
Repository: hbase
Updated Branches:
  refs/heads/master 112900d04 -> 23ec9e290


HBASE-14387 Compaction improvements: Maximum off-peak compaction size (Vladimir Rodionov)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/23ec9e29
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/23ec9e29
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/23ec9e29

Branch: refs/heads/master
Commit: 23ec9e290ebcf71a117ce27b152be84cc561f034
Parents: 112900d
Author: Enis Soztutar <enis@apache.org>
Authored: Mon Nov 9 19:54:29 2015 -0800
Committer: Enis Soztutar <enis@apache.org>
Committed: Tue Nov 10 11:26:52 2015 -0800

----------------------------------------------------------------------
 .../compactions/CompactionConfiguration.java    | 20 +++++++++++++++++++-
 .../compactions/ExploringCompactionPolicy.java  |  2 +-
 .../compactions/RatioBasedCompactionPolicy.java |  7 ++++---
 .../TestRegionServerOnlineConfigChange.java     |  8 ++++++++
 4 files changed, 32 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/23ec9e29/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionConfiguration.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionConfiguration.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionConfiguration.java
index 630ca7d..62e7c7c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionConfiguration.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionConfiguration.java
@@ -57,6 +57,8 @@ public class CompactionConfiguration {
   public static final String HBASE_HSTORE_COMPACTION_MAX_KEY = "hbase.hstore.compaction.max";
   public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY =
     "hbase.hstore.compaction.max.size";
+  public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY =
+      "hbase.hstore.compaction.max.size.offpeak";
   public static final String HBASE_HSTORE_OFFPEAK_END_HOUR = "hbase.offpeak.end.hour";
   public static final String HBASE_HSTORE_OFFPEAK_START_HOUR = "hbase.offpeak.start.hour";
   public static final String HBASE_HSTORE_MIN_LOCALITY_TO_SKIP_MAJOR_COMPACT =
@@ -68,6 +70,7 @@ public class CompactionConfiguration {
   private final double offPeakCompactionRatio;
   /** Since all these properties can change online, they are volatile **/
   private final long maxCompactSize;
+  private final long offPeakMaxCompactSize;
   private final long minCompactSize;
   private final int minFilesToCompact;
   private final int maxFilesToCompact;
@@ -82,6 +85,8 @@ public class CompactionConfiguration {
     this.storeConfigInfo = storeConfigInfo;
 
     maxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, Long.MAX_VALUE);
+    offPeakMaxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY, 
+      maxCompactSize);      
     minCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY,
         storeConfigInfo.getMemstoreFlushSize());
     minFilesToCompact = Math.max(2, conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY,
@@ -102,10 +107,11 @@ public class CompactionConfiguration {
   @Override
   public String toString() {
     return String.format(
-      "size [%d, %d); files [%d, %d); ratio %f; off-peak ratio %f; throttle point %d;"
+      "size [%d, %d, %d); files [%d, %d); ratio %f; off-peak ratio %f; throttle point %d;"
       + " major period %d, major jitter %f, min locality to compact %f",
       minCompactSize,
       maxCompactSize,
+      offPeakMaxCompactSize,
       minFilesToCompact,
       maxFilesToCompact,
       compactionRatio,
@@ -189,4 +195,16 @@ public class CompactionConfiguration {
   public float getMinLocalityToForceCompact() {
     return minLocalityToForceCompact;
   }
+
+  public long getOffPeakMaxCompactSize() {
+    return offPeakMaxCompactSize;
+  }
+  
+  public long getMaxCompactSize(boolean mayUseOffpeak) {
+    if (mayUseOffpeak) {
+      return getOffPeakMaxCompactSize();
+    } else {
+      return getMaxCompactSize();
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/23ec9e29/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
index dbd4adb..c9d911d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
@@ -96,7 +96,7 @@ public class ExploringCompactionPolicy extends RatioBasedCompactionPolicy
{
           smallestSize = size;
         }
 
-        if (size > comConf.getMaxCompactSize()) {
+        if (size > comConf.getMaxCompactSize(mayUseOffPeak)) {
           continue;
         }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/23ec9e29/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
index 5aeff5c..a823d7c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
@@ -96,7 +96,7 @@ public class RatioBasedCompactionPolicy extends CompactionPolicy {
     // If we can't have all files, we cannot do major anyway
     boolean isAllFiles = candidateFiles.size() == candidateSelection.size();
     if (!(forceMajor && isAllFiles)) {
-      candidateSelection = skipLargeFiles(candidateSelection);
+      candidateSelection = skipLargeFiles(candidateSelection, mayUseOffPeak);
       isAllFiles = candidateFiles.size() == candidateSelection.size();
     }
 
@@ -128,10 +128,11 @@ public class RatioBasedCompactionPolicy extends CompactionPolicy {
    * exclude all files above maxCompactSize
    * Also save all references. We MUST compact them
    */
-  private ArrayList<StoreFile> skipLargeFiles(ArrayList<StoreFile> candidates)
{
+  private ArrayList<StoreFile> skipLargeFiles(ArrayList<StoreFile> candidates,

+    boolean mayUseOffpeak) {
     int pos = 0;
     while (pos < candidates.size() && !candidates.get(pos).isReference()
-      && (candidates.get(pos).getReader().length() > comConf.getMaxCompactSize()))
{
+      && (candidates.get(pos).getReader().length() > comConf.getMaxCompactSize(mayUseOffpeak)))
{
       ++pos;
     }
     if (pos > 0) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/23ec9e29/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerOnlineConfigChange.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerOnlineConfigChange.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerOnlineConfigChange.java
index dbb080a..8d0b1d3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerOnlineConfigChange.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerOnlineConfigChange.java
@@ -194,6 +194,14 @@ public class TestRegionServerOnlineConfigChange {
     rs1.getConfigurationManager().notifyAllObservers(conf);
     assertEquals(newMaxCompactSize,
                  hstore.getStoreEngine().getCompactionPolicy().getConf().getMaxCompactSize());
+    // Check if the offPeakMaxCompactSize gets updated.
+    long newOffpeakMaxCompactSize =
+            hstore.getStoreEngine().getCompactionPolicy().getConf().getOffPeakMaxCompactSize()
- 1;
+    conf.setLong(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_SIZE_OFFPEAK_KEY,
+      newOffpeakMaxCompactSize);
+    rs1.getConfigurationManager().notifyAllObservers(conf);
+    assertEquals(newOffpeakMaxCompactSize,
+                 hstore.getStoreEngine().getCompactionPolicy().getConf().getOffPeakMaxCompactSize());
 
     // Check if majorCompactionPeriod gets updated.
     long newMajorCompactionPeriod =


Mime
View raw message