hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject svn commit: r1477391 - in /hbase/trunk/hbase-server/src: main/java/org/apache/hadoop/hbase/regionserver/compactions/ test/java/org/apache/hadoop/hbase/regionserver/compactions/
Date Mon, 29 Apr 2013 22:36:57 GMT
Author: sershe
Date: Mon Apr 29 22:36:56 2013
New Revision: 1477391

URL: http://svn.apache.org/r1477391
Log:
HBASE-8448 RatioBasedCompactionPolicy (and derived ones) can select already-compacting files
for compaction

Modified:
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java?rev=1477391&r1=1477390&r2=1477391&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
Mon Apr 29 22:36:56 2013
@@ -37,8 +37,6 @@ import org.apache.hadoop.hbase.regionser
 @InterfaceAudience.Private
 public class ExploringCompactionPolicy extends RatioBasedCompactionPolicy {
 
-  /** Computed number of files that are needed to assume compactions are stuck. */
-  private final long filesNeededToForce;
 
   /**
    * Constructor for ExploringCompactionPolicy.
@@ -48,20 +46,17 @@ public class ExploringCompactionPolicy e
   public ExploringCompactionPolicy(final Configuration conf,
                                    final StoreConfigInformation storeConfigInfo) {
     super(conf, storeConfigInfo);
-    filesNeededToForce = storeConfigInfo.getBlockingFileCount();
   }
 
   @Override
   final ArrayList<StoreFile> applyCompactionPolicy(final ArrayList<StoreFile>
candidates,
-                                             final boolean mayUseOffPeak) throws IOException
{
+    final boolean mayUseOffPeak, final boolean mightBeStuck) throws IOException {
     // Start off choosing nothing.
     List<StoreFile> bestSelection = new ArrayList<StoreFile>(0);
     List<StoreFile> smallest = new ArrayList<StoreFile>(0);
     long bestSize = 0;
     long smallestSize = Long.MAX_VALUE;
 
-    boolean mightBeStuck = candidates.size() >= filesNeededToForce;
-
     // Consider every starting place.
     for (int start = 0; start < candidates.size(); start++) {
       // Consider every different sub list permutation in between start and end with min
files.

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java?rev=1477391&r1=1477390&r2=1477391&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
Mon Apr 29 22:36:56 2013
@@ -82,6 +82,12 @@ public class RatioBasedCompactionPolicy 
       final boolean mayUseOffPeak, final boolean forceMajor) throws IOException {
     // Preliminary compaction subject to filters
     ArrayList<StoreFile> candidateSelection = new ArrayList<StoreFile>(candidateFiles);
+    // Stuck and not compacting enough (estimate). It is not guaranteed that we will be
+    // able to compact more if stuck and compacting, because ratio policy excludes some
+    // non-compacting files from consideration during compaction (see getCurrentEligibleFiles).
+    int futureFiles = filesCompacting.isEmpty() ? 0 : 1;
+    boolean mayBeStuck = (candidateFiles.size() - filesCompacting.size() + futureFiles)
+        >= storeConfigInfo.getBlockingFileCount();
     candidateSelection = getCurrentEligibleFiles(candidateSelection, filesCompacting);
     long cfTtl = this.storeConfigInfo.getStoreFileTtl();
     if (!forceMajor) {
@@ -110,18 +116,10 @@ public class RatioBasedCompactionPolicy 
     if (!majorCompaction) {
       // we're doing a minor compaction, let's see what files are applicable
       candidateSelection = filterBulk(candidateSelection);
-      candidateSelection = applyCompactionPolicy(candidateSelection, mayUseOffPeak);
+      candidateSelection = applyCompactionPolicy(candidateSelection, mayUseOffPeak, mayBeStuck);
       candidateSelection = checkMinFilesCriteria(candidateSelection);
     }
     candidateSelection = removeExcessFiles(candidateSelection, isUserCompaction, majorCompaction);
-
-    if (candidateSelection.size() == 0
-        && candidateFiles.size() >= storeConfigInfo.getBlockingFileCount()) {
-      candidateSelection = new ArrayList<StoreFile>(candidateFiles);
-      candidateSelection
-          .subList(0, Math.max(0,candidateSelection.size() - comConf.getMinFilesToCompact()))
-          .clear();
-    }
     CompactionRequest result = new CompactionRequest(candidateSelection);
     result.setOffPeak(!candidateSelection.isEmpty() && !majorCompaction &&
mayUseOffPeak);
     return result;
@@ -261,8 +259,8 @@ public class RatioBasedCompactionPolicy 
     *    | |  | |  | |  | | | | | |
     *    | |  | |  | |  | | | | | |
     */
-  ArrayList<StoreFile> applyCompactionPolicy(
-      ArrayList<StoreFile> candidates, boolean mayUseOffPeak) throws IOException {
+  ArrayList<StoreFile> applyCompactionPolicy(ArrayList<StoreFile> candidates,
+      boolean mayUseOffPeak, boolean mayBeStuck) throws IOException {
     if (candidates.isEmpty()) {
       return candidates;
     }
@@ -298,11 +296,14 @@ public class RatioBasedCompactionPolicy 
     if (start < countOfFiles) {
       LOG.info("Default compaction algorithm has selected " + (countOfFiles - start)
         + " files from " + countOfFiles + " candidates");
+    } else if (mayBeStuck) {
+      // We may be stuck. Compact the latest files if we can.
+      int filesToLeave = candidates.size() - comConf.getMinFilesToCompact();
+      if (filesToLeave >= 0) {
+        start = filesToLeave;
+      }
     }
-
-    if (start > 0) {
-      candidates.subList(0, start).clear();
-    }
+    candidates.subList(0, start).clear();
     return candidates;
   }
 

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java?rev=1477391&r1=1477390&r2=1477391&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java
Mon Apr 29 22:36:56 2013
@@ -43,7 +43,7 @@ public class EverythingPolicy extends Ra
 
   @Override
   final ArrayList<StoreFile> applyCompactionPolicy(final ArrayList<StoreFile>
candidates,
-                                                   final boolean mayUseOffPeak) throws IOException
{
+    final boolean mayUseOffPeak, final boolean mayBeStuck) throws IOException {
 
     if (candidates.size() < comConf.getMinFilesToCompact()) {
       return new ArrayList<StoreFile>(0);



Mime
View raw message