hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ecl...@apache.org
Subject svn commit: r1465135 - in /hbase/branches/0.95/hbase-server/src: main/java/org/apache/hadoop/hbase/regionserver/ main/java/org/apache/hadoop/hbase/regionserver/compactions/ test/java/org/apache/hadoop/hbase/regionserver/ test/java/org/apache/hadoop/hba...
Date Fri, 05 Apr 2013 21:27:01 GMT
Author: eclark
Date: Fri Apr  5 21:27:00 2013
New Revision: 1465135

URL: http://svn.apache.org/r1465135
Log:
HBASE-7842 Add compaction policy that explores more storefile groups

Added:
    hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
    hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
Removed:
    hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactionPolicy.java
Modified:
    hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java
    hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java
    hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
    hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
    hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java
    hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/PerfTestCompactionPolicies.java

Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java?rev=1465135&r1=1465134&r2=1465135&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java
(original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java
Fri Apr  5 21:27:00 2013
@@ -19,7 +19,6 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -27,11 +26,10 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.KeyValue.KVComparator;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactionPolicy;
-import org.apache.hadoop.hbase.regionserver.compactions.Compactor;
-import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactionPolicy;
+import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
 import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
 import org.apache.hadoop.hbase.util.ReflectionUtils;
+import org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy;
 
 /**
  * Default StoreEngine creates the default compactor, policy, and store file manager, or
@@ -39,7 +37,7 @@ import org.apache.hadoop.hbase.util.Refl
  */
 @InterfaceAudience.Private
 public class DefaultStoreEngine extends StoreEngine<
-  DefaultCompactionPolicy, DefaultCompactor, DefaultStoreFileManager> {
+    RatioBasedCompactionPolicy, DefaultCompactor, DefaultStoreFileManager> {
 
   public static final String DEFAULT_COMPACTOR_CLASS_KEY =
       "hbase.hstore.defaultengine.compactor.class";
@@ -48,8 +46,8 @@ public class DefaultStoreEngine extends 
 
   private static final Class<? extends DefaultCompactor>
     DEFAULT_COMPACTOR_CLASS = DefaultCompactor.class;
-  private static final Class<? extends DefaultCompactionPolicy>
-    DEFAULT_COMPACTION_POLICY_CLASS = DefaultCompactionPolicy.class;
+  private static final Class<? extends RatioBasedCompactionPolicy>
+    DEFAULT_COMPACTION_POLICY_CLASS = ExploringCompactionPolicy.class;
 
   @Override
   protected void createComponents(
@@ -98,4 +96,5 @@ public class DefaultStoreEngine extends 
           storeFileManager.getStorefiles(), filesCompacting);
     }
   }
+
 }

Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java?rev=1465135&r1=1465134&r2=1465135&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java
(original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java
Fri Apr  5 21:27:00 2013
@@ -27,8 +27,6 @@ import org.apache.hadoop.hbase.KeyValue.
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionPolicy;
 import org.apache.hadoop.hbase.regionserver.compactions.Compactor;
-import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactionPolicy;
-import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
 import org.apache.hadoop.hbase.util.ReflectionUtils;
 
 /**

Added: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java?rev=1465135&view=auto
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
(added)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
Fri Apr  5 21:27:00 2013
@@ -0,0 +1,89 @@
+package org.apache.hadoop.hbase.regionserver.compactions;
+
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+
+@InterfaceAudience.Private
+public class ExploringCompactionPolicy extends RatioBasedCompactionPolicy {
+
+  public ExploringCompactionPolicy(Configuration conf,
+                                   StoreConfigInformation storeConfigInfo) {
+    super(conf, storeConfigInfo);
+  }
+
+  @Override
+  ArrayList<StoreFile> applyCompactionPolicy(ArrayList<StoreFile> candidates,
+                                             boolean mayUseOffPeak) throws IOException {
+    // Start off choosing nothing.
+    List<StoreFile> bestSelection = new ArrayList<StoreFile>(0);
+    long bestSize = 0;
+
+    // Consider every starting place.
+    for (int start = 0; start < candidates.size(); start++) {
+      // Consider every different sub list permutation in between start and end with min
files.
+      for(int currentEnd = start + comConf.getMinFilesToCompact() - 1;
+          currentEnd < candidates.size(); currentEnd++) {
+        List<StoreFile> potentialMatchFiles = candidates.subList(start, currentEnd+1);
+
+        // Sanity checks
+        if (potentialMatchFiles.size() < comConf.getMinFilesToCompact()) continue;
+        if (potentialMatchFiles.size() > comConf.getMaxFilesToCompact()) continue;
+        if (!filesInRatio(potentialMatchFiles, mayUseOffPeak)) continue;
+
+        // Compute the total size of files that will
+        // have to be read if this set of files is compacted.
+        long size = 0;
+
+        for (StoreFile s:potentialMatchFiles) {
+          size += s.getReader().length();
+        }
+
+        // Keep if this gets rid of more files.  Or the same number of files for less io.
+        if (potentialMatchFiles.size() > bestSelection.size() ||
+            (potentialMatchFiles.size() == bestSelection.size() && size < bestSize))
{
+          bestSelection = potentialMatchFiles;
+          bestSize = size;
+        }
+      }
+    }
+
+    return new ArrayList<StoreFile>(bestSelection);
+  }
+
+  /**
+   * Check that all files satisfy the r
+   * @param files
+   * @return
+   */
+  private boolean filesInRatio(List<StoreFile> files, boolean isOffPeak) {
+    if (files.size() < 2) {
+      return  true;
+    }
+    double currentRatio = isOffPeak ?
+                          comConf.getCompactionRatioOffPeak() : comConf.getCompactionRatio();
+
+    long totalFileSize = 0;
+    for (int i = 0; i < files.size(); i++) {
+      totalFileSize += files.get(i).getReader().length();
+    }
+    for (int i = 0; i < files.size(); i++) {
+      long singleFileSize = files.get(i).getReader().length();
+      long sumAllOtherFilesize = totalFileSize - singleFileSize;
+
+      if (( singleFileSize >  sumAllOtherFilesize * currentRatio)
+          && (sumAllOtherFilesize >= comConf.getMinCompactSize())){
+        return false;
+      }
+    }
+
+    return true;
+
+  }
+}

Added: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java?rev=1465135&view=auto
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
(added)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
Fri Apr  5 21:27:00 2013
@@ -0,0 +1,384 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver.compactions;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.StoreUtils;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Collections2;
+
+/**
+ * The default algorithm for selecting files for compaction.
+ * Combines the compaction configuration and the provisional file selection that
+ * it's given to produce the list of suitable candidates for compaction.
+ */
+@InterfaceAudience.Private
+public class RatioBasedCompactionPolicy extends CompactionPolicy {
+  private static final Log LOG = LogFactory.getLog(RatioBasedCompactionPolicy.class);
+
+  public RatioBasedCompactionPolicy(Configuration conf,
+                                    StoreConfigInformation storeConfigInfo) {
+    super(conf, storeConfigInfo);
+  }
+
+  private ArrayList<StoreFile> getCurrentEligibleFiles(
+      ArrayList<StoreFile> candidateFiles, final List<StoreFile> filesCompacting)
{
+    // candidates = all storefiles not already in compaction queue
+    if (!filesCompacting.isEmpty()) {
+      // exclude all files older than the newest file we're currently
+      // compacting. this allows us to preserve contiguity (HBASE-2856)
+      StoreFile last = filesCompacting.get(filesCompacting.size() - 1);
+      int idx = candidateFiles.indexOf(last);
+      Preconditions.checkArgument(idx != -1);
+      candidateFiles.subList(0, idx + 1).clear();
+    }
+    return candidateFiles;
+  }
+
+  public List<StoreFile> preSelectCompactionForCoprocessor(
+      final Collection<StoreFile> candidates, final List<StoreFile> filesCompacting)
{
+    return getCurrentEligibleFiles(new ArrayList<StoreFile>(candidates), filesCompacting);
+  }
+
+  /**
+   * @param candidateFiles candidate files, ordered from oldest to newest
+   * @return subset copy of candidate list that meets compaction criteria
+   * @throws java.io.IOException
+   */
+  public CompactionRequest selectCompaction(Collection<StoreFile> candidateFiles,
+      final List<StoreFile> filesCompacting, final boolean isUserCompaction,
+      final boolean mayUseOffPeak, final boolean forceMajor) throws IOException {
+    // Preliminary compaction subject to filters
+    ArrayList<StoreFile> candidateSelection = new ArrayList<StoreFile>(candidateFiles);
+    candidateSelection = getCurrentEligibleFiles(candidateSelection, filesCompacting);
+    long cfTtl = this.storeConfigInfo.getStoreFileTtl();
+    if (!forceMajor) {
+      // If there are expired files, only select them so that compaction deletes them
+      if (comConf.shouldDeleteExpired() && (cfTtl != Long.MAX_VALUE)) {
+        ArrayList<StoreFile> expiredSelection = selectExpiredStoreFiles(
+            candidateSelection, EnvironmentEdgeManager.currentTimeMillis() - cfTtl);
+        if (expiredSelection != null) {
+          return new CompactionRequest(expiredSelection);
+        }
+      }
+      candidateSelection = skipLargeFiles(candidateSelection);
+    }
+
+    // Force a major compaction if this is a user-requested major compaction,
+    // or if we do not have too many files to compact and this was requested
+    // as a major compaction.
+    // Or, if there are any references among the candidates.
+    boolean majorCompaction = (
+      (forceMajor && isUserCompaction)
+      || ((forceMajor || isMajorCompaction(candidateSelection))
+          && (candidateSelection.size() < comConf.getMaxFilesToCompact()))
+      || StoreUtils.hasReferences(candidateSelection)
+      );
+
+    if (!majorCompaction) {
+      // we're doing a minor compaction, let's see what files are applicable
+      candidateSelection = filterBulk(candidateSelection);
+      candidateSelection = applyCompactionPolicy(candidateSelection, mayUseOffPeak);
+      candidateSelection = checkMinFilesCriteria(candidateSelection);
+    }
+    candidateSelection = removeExcessFiles(candidateSelection, isUserCompaction, majorCompaction);
+    CompactionRequest result = new CompactionRequest(candidateSelection);
+    result.setOffPeak(!candidateSelection.isEmpty() && !majorCompaction &&
mayUseOffPeak);
+    return result;
+  }
+
+  /**
+   * Select the expired store files to compact
+   *
+   * @param candidates the initial set of storeFiles
+   * @param maxExpiredTimeStamp
+   *          The store file will be marked as expired if its max time stamp is
+   *          less than this maxExpiredTimeStamp.
+   * @return A CompactSelection contains the expired store files as
+   *         filesToCompact
+   */
+  private ArrayList<StoreFile> selectExpiredStoreFiles(
+      ArrayList<StoreFile> candidates, long maxExpiredTimeStamp) {
+    if (candidates == null || candidates.size() == 0) return null;
+    ArrayList<StoreFile> expiredStoreFiles = null;
+
+    for (StoreFile storeFile : candidates) {
+      if (storeFile.getReader().getMaxTimestamp() < maxExpiredTimeStamp) {
+        LOG.info("Deleting the expired store file by compaction: "
+            + storeFile.getPath() + " whose maxTimeStamp is "
+            + storeFile.getReader().getMaxTimestamp()
+            + " while the max expired timestamp is " + maxExpiredTimeStamp);
+        if (expiredStoreFiles == null) {
+          expiredStoreFiles = new ArrayList<StoreFile>();
+        }
+        expiredStoreFiles.add(storeFile);
+      }
+    }
+
+    return expiredStoreFiles;
+  }
+
+  /**
+   * @param candidates pre-filtrate
+   * @return filtered subset
+   * exclude all files above maxCompactSize
+   * Also save all references. We MUST compact them
+   */
+  private ArrayList<StoreFile> skipLargeFiles(ArrayList<StoreFile> candidates)
{
+    int pos = 0;
+    while (pos < candidates.size() && !candidates.get(pos).isReference()
+      && (candidates.get(pos).getReader().length() > comConf.getMaxCompactSize()))
{
+      ++pos;
+    }
+    if (pos > 0) {
+      LOG.debug("Some files are too large. Excluding " + pos
+          + " files from compaction candidates");
+      candidates.subList(0, pos).clear();
+    }
+    return candidates;
+  }
+
+  /**
+   * @param candidates pre-filtrate
+   * @return filtered subset
+   * exclude all bulk load files if configured
+   */
+  private ArrayList<StoreFile> filterBulk(ArrayList<StoreFile> candidates) {
+    candidates.removeAll(Collections2.filter(candidates,
+        new Predicate<StoreFile>() {
+          @Override
+          public boolean apply(StoreFile input) {
+            return input.excludeFromMinorCompaction();
+          }
+        }));
+    return candidates;
+  }
+
+  /**
+   * @param candidates pre-filtrate
+   * @return filtered subset
+   * take upto maxFilesToCompact from the start
+   */
+  private ArrayList<StoreFile> removeExcessFiles(ArrayList<StoreFile> candidates,
+      boolean isUserCompaction, boolean isMajorCompaction) {
+    int excess = candidates.size() - comConf.getMaxFilesToCompact();
+    if (excess > 0) {
+      if (isMajorCompaction && isUserCompaction) {
+        LOG.debug("Warning, compacting more than " + comConf.getMaxFilesToCompact() +
+            " files because of a user-requested major compaction");
+      } else {
+        LOG.debug("Too many admissible files. Excluding " + excess
+          + " files from compaction candidates");
+        candidates.subList(comConf.getMaxFilesToCompact(), candidates.size()).clear();
+      }
+    }
+    return candidates;
+  }
+  /**
+   * @param candidates pre-filtrate
+   * @return filtered subset
+   * forget the compactionSelection if we don't have enough files
+   */
+  private ArrayList<StoreFile> checkMinFilesCriteria(ArrayList<StoreFile> candidates)
{
+    int minFiles = comConf.getMinFilesToCompact();
+    if (candidates.size() < minFiles) {
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("Not compacting files because we only have " + candidates.size() +
+          " files ready for compaction. Need " + minFiles + " to initiate.");
+      }
+      candidates.clear();
+    }
+    return candidates;
+  }
+
+  /**
+    * @param candidates pre-filtrate
+    * @return filtered subset
+    * -- Default minor compaction selection algorithm:
+    * choose CompactSelection from candidates --
+    * First exclude bulk-load files if indicated in configuration.
+    * Start at the oldest file and stop when you find the first file that
+    * meets compaction criteria:
+    * (1) a recently-flushed, small file (i.e. <= minCompactSize)
+    * OR
+    * (2) within the compactRatio of sum(newer_files)
+    * Given normal skew, any newer files will also meet this criteria
+    * <p/>
+    * Additional Note:
+    * If fileSizes.size() >> maxFilesToCompact, we will recurse on
+    * compact().  Consider the oldest files first to avoid a
+    * situation where we always compact [end-threshold,end).  Then, the
+    * last file becomes an aggregate of the previous compactions.
+    *
+    * normal skew:
+    *
+    *         older ----> newer (increasing seqID)
+    *     _
+    *    | |   _
+    *    | |  | |   _
+    *  --|-|- |-|- |-|---_-------_-------  minCompactSize
+    *    | |  | |  | |  | |  _  | |
+    *    | |  | |  | |  | | | | | |
+    *    | |  | |  | |  | | | | | |
+    */
+  ArrayList<StoreFile> applyCompactionPolicy(
+      ArrayList<StoreFile> candidates, boolean mayUseOffPeak) throws IOException {
+    if (candidates.isEmpty()) {
+      return candidates;
+    }
+
+    // we're doing a minor compaction, let's see what files are applicable
+    int start = 0;
+    double ratio = comConf.getCompactionRatio();
+    if (mayUseOffPeak) {
+      ratio = comConf.getCompactionRatioOffPeak();
+      LOG.info("Running an off-peak compaction, selection ratio = " + ratio);
+    }
+
+    // get store file sizes for incremental compacting selection.
+    final int countOfFiles = candidates.size();
+    long[] fileSizes = new long[countOfFiles];
+    long[] sumSize = new long[countOfFiles];
+    for (int i = countOfFiles - 1; i >= 0; --i) {
+      StoreFile file = candidates.get(i);
+      fileSizes[i] = file.getReader().length();
+      // calculate the sum of fileSizes[i,i+maxFilesToCompact-1) for algo
+      int tooFar = i + comConf.getMaxFilesToCompact() - 1;
+      sumSize[i] = fileSizes[i]
+        + ((i + 1 < countOfFiles) ? sumSize[i + 1] : 0)
+        - ((tooFar < countOfFiles) ? fileSizes[tooFar] : 0);
+    }
+
+
+    while (countOfFiles - start >= comConf.getMinFilesToCompact() &&
+      fileSizes[start] > Math.max(comConf.getMinCompactSize(),
+          (long) (sumSize[start + 1] * ratio))) {
+      ++start;
+    }
+    if (start < countOfFiles) {
+      LOG.info("Default compaction algorithm has selected " + (countOfFiles - start)
+        + " files from " + countOfFiles + " candidates");
+    }
+
+    if (start > 0) {
+      candidates.subList(0, start).clear();
+    }
+    return candidates;
+  }
+
+  /*
+   * @param filesToCompact Files to compact. Can be null.
+   * @return True if we should run a major compaction.
+   */
+  public boolean isMajorCompaction(final Collection<StoreFile> filesToCompact)
+      throws IOException {
+    boolean result = false;
+    long mcTime = getNextMajorCompactTime(filesToCompact);
+    if (filesToCompact == null || filesToCompact.isEmpty() || mcTime == 0) {
+      return result;
+    }
+    // TODO: Use better method for determining stamp of last major (HBASE-2990)
+    long lowTimestamp = StoreUtils.getLowestTimestamp(filesToCompact);
+    long now = System.currentTimeMillis();
+    if (lowTimestamp > 0l && lowTimestamp < (now - mcTime)) {
+      // Major compaction time has elapsed.
+      long cfTtl = this.storeConfigInfo.getStoreFileTtl();
+      if (filesToCompact.size() == 1) {
+        // Single file
+        StoreFile sf = filesToCompact.iterator().next();
+        Long minTimestamp = sf.getMinimumTimestamp();
+        long oldest = (minTimestamp == null)
+            ? Long.MIN_VALUE
+            : now - minTimestamp.longValue();
+        if (sf.isMajorCompaction() &&
+            (cfTtl == HConstants.FOREVER || oldest < cfTtl)) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Skipping major compaction of " + this +
+                " because one (major) compacted file only and oldestTime " +
+                oldest + "ms is < ttl=" + cfTtl);
+          }
+        } else if (cfTtl != HConstants.FOREVER && oldest > cfTtl) {
+          LOG.debug("Major compaction triggered on store " + this +
+            ", because keyvalues outdated; time since last major compaction " +
+            (now - lowTimestamp) + "ms");
+          result = true;
+        }
+      } else {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Major compaction triggered on store " + this +
+              "; time since last major compaction " + (now - lowTimestamp) + "ms");
+        }
+        result = true;
+      }
+    }
+    return result;
+  }
+
+  public long getNextMajorCompactTime(final Collection<StoreFile> filesToCompact) {
+    // default = 24hrs
+    long ret = comConf.getMajorCompactionPeriod();
+    if (ret > 0) {
+      // default = 20% = +/- 4.8 hrs
+      double jitterPct = comConf.getMajorCompactionJitter();
+      if (jitterPct > 0) {
+        long jitter = Math.round(ret * jitterPct);
+        // deterministic jitter avoids a major compaction storm on restart
+        Integer seed = StoreUtils.getDeterministicRandomSeed(filesToCompact);
+        if (seed != null) {
+          double rnd = (new Random(seed)).nextDouble();
+          ret += jitter - Math.round(2L * jitter * rnd);
+        } else {
+          ret = 0; // no storefiles == no major compaction
+        }
+      }
+    }
+    return ret;
+  }
+
+  /**
+   * @param compactionSize Total size of some compaction
+   * @return whether this should be a large or small compaction
+   */
+  public boolean throttleCompaction(long compactionSize) {
+    return compactionSize > comConf.getThrottlePoint();
+  }
+
+  @Override
+  public boolean needsCompaction(final Collection<StoreFile> storeFiles,
+      final List<StoreFile> filesCompacting) {
+    int numCandidates = storeFiles.size() - filesCompacting.size();
+    return numCandidates > comConf.getMinFilesToCompact();
+  }
+}

Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java?rev=1465135&r1=1465134&r2=1465135&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
(original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
Fri Apr  5 21:27:00 2013
@@ -55,7 +55,7 @@ import org.apache.hadoop.hbase.io.hfile.
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 import org.apache.hadoop.hbase.regionserver.compactions.Compactor;
-import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactionPolicy;
+import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
@@ -321,7 +321,8 @@ public class TestCompaction extends HBas
       assertEquals(2, s.getStorefilesCount());
 
       // ensure that major compaction time is deterministic
-      DefaultCompactionPolicy c = (DefaultCompactionPolicy)s.storeEngine.getCompactionPolicy();
+      RatioBasedCompactionPolicy
+          c = (RatioBasedCompactionPolicy)s.storeEngine.getCompactionPolicy();
       Collection<StoreFile> storeFiles = s.getStorefiles();
       long mcTime = c.getNextMajorCompactTime(storeFiles);
       for (int i = 0; i < 10; ++i) {

Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java?rev=1465135&r1=1465134&r2=1465135&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
(original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
Fri Apr  5 21:27:00 2013
@@ -20,8 +20,6 @@ package org.apache.hadoop.hbase.regionse
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Calendar;
-import java.util.GregorianCalendar;
 import java.util.List;
 
 import junit.framework.TestCase;
@@ -40,7 +38,7 @@ import org.apache.hadoop.hbase.SmallTest
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
-import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactionPolicy;
+import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -171,6 +169,16 @@ public class TestDefaultCompactSelection
         }
       };
     }
+
+    @Override
+    public String toString() {
+      return "MockStoreFile{" +
+          "length=" + length +
+          ", isRef=" + isRef +
+          ", ageInDisk=" + ageInDisk +
+          ", sequenceid=" + sequenceid +
+          '}';
+    }
   }
 
   ArrayList<Long> toArrayList(long... numbers) {
@@ -234,7 +242,7 @@ public class TestDefaultCompactSelection
   throws IOException {
     store.forceMajor = forcemajor;
     //Test Default compactions
-    CompactionRequest result = ((DefaultCompactionPolicy)store.storeEngine.getCompactionPolicy())
+    CompactionRequest result = ((RatioBasedCompactionPolicy)store.storeEngine.getCompactionPolicy())
         .selectCompaction(candidates, new ArrayList<StoreFile>(), false, isOffPeak,
forcemajor);
     List<StoreFile> actual = new ArrayList<StoreFile>(result.getFiles());
     if (isOffPeak && !forcemajor) {
@@ -269,7 +277,13 @@ public class TestDefaultCompactSelection
      */
     // don't exceed max file compact threshold
     // note:  file selection starts with largest to smallest.
-    compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 7, 6, 5, 4, 3);
+    compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 5, 4, 3, 2, 1);
+
+    compactEquals(sfCreate(50, 10, 10 ,10, 10), 10, 10, 10, 10);
+
+    compactEquals(sfCreate(10, 10, 10, 10, 50), 10, 10, 10, 10);
+
+    compactEquals(sfCreate(251, 253, 251, maxSize -1), 251, 253, 251);
 
     /* MAJOR COMPACTION */
     // if a major compaction has been forced, then compact everything
@@ -280,7 +294,7 @@ public class TestDefaultCompactSelection
     compactEquals(sfCreate(tooBig, 12,12), true, tooBig, 12, 12);
     // don't exceed max file compact threshold, even with major compaction
     store.forceMajor = true;
-    compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 7, 6, 5, 4, 3);
+    compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 5, 4, 3, 2, 1);
     store.forceMajor = false;
     // if we exceed maxCompactSize, downgrade to minor
     // if not, it creates a 'snowball effect' when files >> maxCompactSize:

Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java?rev=1465135&r1=1465134&r2=1465135&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java
(original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java
Fri Apr  5 21:27:00 2013
@@ -23,7 +23,7 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.KeyValue.KVComparator;
 import org.apache.hadoop.hbase.SmallTests;
-import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactionPolicy;
+import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
 import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
 import org.junit.Assert;
 import org.junit.Test;
@@ -38,7 +38,7 @@ public class TestDefaultStoreEngine {
     }
   }
 
-  public static class DummyCompactionPolicy extends DefaultCompactionPolicy {
+  public static class DummyCompactionPolicy extends RatioBasedCompactionPolicy {
     public DummyCompactionPolicy(Configuration conf, StoreConfigInformation storeConfigInfo)
{
       super(conf, storeConfigInfo);
     }

Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/PerfTestCompactionPolicies.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/PerfTestCompactionPolicies.java?rev=1465135&r1=1465134&r2=1465135&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/PerfTestCompactionPolicies.java
(original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/PerfTestCompactionPolicies.java
Fri Apr  5 21:27:00 2013
@@ -26,7 +26,6 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.SmallTests;
-import org.apache.hadoop.hbase.KeyValue.KVComparator;
 import org.apache.hadoop.hbase.regionserver.HStore;
 import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
@@ -53,7 +52,7 @@ public class PerfTestCompactionPolicies 
 
   static final Log LOG = LogFactory.getLog(PerfTestCompactionPolicies.class);
 
-  private final DefaultCompactionPolicy cp;
+  private final RatioBasedCompactionPolicy cp;
   private final int max;
   private final int min;
   private final float ratio;
@@ -64,11 +63,16 @@ public class PerfTestCompactionPolicies 
   @Parameterized.Parameters
   public static Collection<Object[]> data() {
     return Arrays.asList(new Object[][] {
-        {DefaultCompactionPolicy.class, 3, 2, 1.2f},
-        {DefaultCompactionPolicy.class, 4, 2, 1.2f},
-        {DefaultCompactionPolicy.class, 5, 2, 1.2f},
-        {DefaultCompactionPolicy.class, 4, 2, 1.3f},
-        {DefaultCompactionPolicy.class, 4, 2, 1.4f},
+        {RatioBasedCompactionPolicy.class, 3, 2, 1.2f},
+        {ExploringCompactionPolicy.class, 3, 2, 1.2f},
+        {RatioBasedCompactionPolicy.class, 4, 2, 1.2f},
+        {ExploringCompactionPolicy.class, 4, 2, 1.2f},
+        {RatioBasedCompactionPolicy.class, 5, 2, 1.2f},
+        {ExploringCompactionPolicy.class, 5, 2, 1.2f},
+        {RatioBasedCompactionPolicy.class, 4, 2, 1.3f},
+        {ExploringCompactionPolicy.class, 4, 2, 1.3f},
+        {RatioBasedCompactionPolicy.class, 4, 2, 1.4f},
+        {ExploringCompactionPolicy.class, 4, 2, 1.4f},
 
     });
   }



Mime
View raw message