hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject svn commit: r1487948 - in /hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver: CompactSplitThread.java compactions/ExploringCompactionPolicy.java compactions/RatioBasedCompactionPolicy.java
Date Thu, 30 May 2013 17:55:04 GMT
Author: sershe
Date: Thu May 30 17:55:04 2013
New Revision: 1487948

URL: http://svn.apache.org/r1487948
Log:
HBASE-8638 add logging to compaction policy

Modified:
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java?rev=1487948&r1=1487947&r2=1487948&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
Thu May 30 17:55:04 2013
@@ -19,6 +19,8 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
@@ -395,7 +397,11 @@ public class CompactSplitThread implemen
           }
         }
       } catch (IOException ex) {
-        LOG.error("Compaction failed " + this, RemoteExceptionHandler.checkIOException(ex));
+        IOException remoteEx = RemoteExceptionHandler.checkIOException(ex);
+        LOG.error("Compaction failed " + this, remoteEx);
+        if (remoteEx != ex) {
+          LOG.info("Compaction failed at original callstack: " + formatStackTrace(ex));
+        }
         server.checkFileSystem();
       } catch (Exception ex) {
         LOG.error("Compaction failed " + this, ex);
@@ -406,6 +412,14 @@ public class CompactSplitThread implemen
       this.compaction.getRequest().afterExecute();
     }
 
+    private String formatStackTrace(Exception ex) {
+      StringWriter sw = new StringWriter();
+      PrintWriter pw = new PrintWriter(sw);
+      ex.printStackTrace(pw);
+      pw.flush();
+      return sw.toString();
+    }
+
     @Override
     public int compareTo(CompactionRunner o) {
       // Only compare the underlying request, for queue sorting purposes.

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java?rev=1487948&r1=1487947&r2=1487948&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
Thu May 30 17:55:04 2013
@@ -23,6 +23,8 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
@@ -36,7 +38,7 @@ import org.apache.hadoop.hbase.regionser
  */
 @InterfaceAudience.Private
 public class ExploringCompactionPolicy extends RatioBasedCompactionPolicy {
-
+  private static final Log LOG = LogFactory.getLog(ExploringCompactionPolicy.class);
 
   /**
    * Constructor for ExploringCompactionPolicy.
@@ -57,6 +59,7 @@ public class ExploringCompactionPolicy e
     long bestSize = 0;
     long smallestSize = Long.MAX_VALUE;
 
+    int opts = 0, optsInRatio = 0, bestStart = -1; // for debug logging
     // Consider every starting place.
     for (int start = 0; start < candidates.size(); start++) {
       // Consider every different sub list permutation in between start and end with min
files.
@@ -83,26 +86,34 @@ public class ExploringCompactionPolicy e
           smallestSize = size;
         }
 
-        if (size >= comConf.getMinCompactSize()
-            && !filesInRatio(potentialMatchFiles, mayUseOffPeak)) {
+        if (size > comConf.getMaxCompactSize()) {
           continue;
         }
 
-        if (size > comConf.getMaxCompactSize()) {
+        ++opts;
+        if (size >= comConf.getMinCompactSize()
+            && !filesInRatio(potentialMatchFiles, mayUseOffPeak)) {
           continue;
         }
 
+        ++optsInRatio;
         // Keep if this gets rid of more files.  Or the same number of files for less io.
         if (potentialMatchFiles.size() > bestSelection.size()
             || (potentialMatchFiles.size() == bestSelection.size() && size < bestSize))
{
           bestSelection = potentialMatchFiles;
           bestSize = size;
+          bestStart = start;
         }
       }
     }
     if (bestSelection.size() == 0 && mightBeStuck) {
+      LOG.debug("Exploring compaction algorithm has selected " + smallest.size()
+          + " files of size "+ smallestSize + " because the store might be stuck");
       return new ArrayList<StoreFile>(smallest);
     }
+    LOG.debug("Exploring compaction algorithm has selected " + bestSelection.size()
+        + " files of size " + bestSize + " starting at candidate #" + bestStart +
+        " after considering " + opts + " permutations with " + optsInRatio + " in ratio");
     return new ArrayList<StoreFile>(bestSelection);
   }
 

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java?rev=1487948&r1=1487947&r2=1487948&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
Thu May 30 17:55:04 2013
@@ -89,6 +89,10 @@ public class RatioBasedCompactionPolicy 
     boolean mayBeStuck = (candidateFiles.size() - filesCompacting.size() + futureFiles)
         >= storeConfigInfo.getBlockingFileCount();
     candidateSelection = getCurrentEligibleFiles(candidateSelection, filesCompacting);
+    LOG.debug("Selecting compaction from " + candidateFiles.size() + " store files, " +
+        filesCompacting.size() + " compacting, " + candidateSelection.size() +
+        " eligible, " + storeConfigInfo.getBlockingFileCount() + " blocking");
+
     long cfTtl = this.storeConfigInfo.getStoreFileTtl();
     if (!forceMajor) {
       // If there are expired files, only select them so that compaction deletes them



Mime
View raw message