Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 5FAE0103CC for ; Mon, 29 Apr 2013 22:37:22 +0000 (UTC) Received: (qmail 62884 invoked by uid 500); 29 Apr 2013 22:37:22 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 62837 invoked by uid 500); 29 Apr 2013 22:37:22 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 62830 invoked by uid 99); 29 Apr 2013 22:37:22 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 29 Apr 2013 22:37:22 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 29 Apr 2013 22:37:18 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 7FA452388A6C; Mon, 29 Apr 2013 22:36:57 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1477391 - in /hbase/trunk/hbase-server/src: main/java/org/apache/hadoop/hbase/regionserver/compactions/ test/java/org/apache/hadoop/hbase/regionserver/compactions/ Date: Mon, 29 Apr 2013 22:36:57 -0000 To: commits@hbase.apache.org From: sershe@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130429223657.7FA452388A6C@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: sershe Date: Mon Apr 29 22:36:56 2013 New Revision: 1477391 URL: http://svn.apache.org/r1477391 Log: HBASE-8448 RatioBasedCompactionPolicy (and derived ones) can select already-compacting files for compaction Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java?rev=1477391&r1=1477390&r2=1477391&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java Mon Apr 29 22:36:56 2013 @@ -37,8 +37,6 @@ import org.apache.hadoop.hbase.regionser @InterfaceAudience.Private public class ExploringCompactionPolicy extends RatioBasedCompactionPolicy { - /** Computed number of files that are needed to assume compactions are stuck. */ - private final long filesNeededToForce; /** * Constructor for ExploringCompactionPolicy. @@ -48,20 +46,17 @@ public class ExploringCompactionPolicy e public ExploringCompactionPolicy(final Configuration conf, final StoreConfigInformation storeConfigInfo) { super(conf, storeConfigInfo); - filesNeededToForce = storeConfigInfo.getBlockingFileCount(); } @Override final ArrayList applyCompactionPolicy(final ArrayList candidates, - final boolean mayUseOffPeak) throws IOException { + final boolean mayUseOffPeak, final boolean mightBeStuck) throws IOException { // Start off choosing nothing. List bestSelection = new ArrayList(0); List smallest = new ArrayList(0); long bestSize = 0; long smallestSize = Long.MAX_VALUE; - boolean mightBeStuck = candidates.size() >= filesNeededToForce; - // Consider every starting place. for (int start = 0; start < candidates.size(); start++) { // Consider every different sub list permutation in between start and end with min files. Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java?rev=1477391&r1=1477390&r2=1477391&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java (original) +++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java Mon Apr 29 22:36:56 2013 @@ -82,6 +82,12 @@ public class RatioBasedCompactionPolicy final boolean mayUseOffPeak, final boolean forceMajor) throws IOException { // Preliminary compaction subject to filters ArrayList candidateSelection = new ArrayList(candidateFiles); + // Stuck and not compacting enough (estimate). It is not guaranteed that we will be + // able to compact more if stuck and compacting, because ratio policy excludes some + // non-compacting files from consideration during compaction (see getCurrentEligibleFiles). + int futureFiles = filesCompacting.isEmpty() ? 0 : 1; + boolean mayBeStuck = (candidateFiles.size() - filesCompacting.size() + futureFiles) + >= storeConfigInfo.getBlockingFileCount(); candidateSelection = getCurrentEligibleFiles(candidateSelection, filesCompacting); long cfTtl = this.storeConfigInfo.getStoreFileTtl(); if (!forceMajor) { @@ -110,18 +116,10 @@ public class RatioBasedCompactionPolicy if (!majorCompaction) { // we're doing a minor compaction, let's see what files are applicable candidateSelection = filterBulk(candidateSelection); - candidateSelection = applyCompactionPolicy(candidateSelection, mayUseOffPeak); + candidateSelection = applyCompactionPolicy(candidateSelection, mayUseOffPeak, mayBeStuck); candidateSelection = checkMinFilesCriteria(candidateSelection); } candidateSelection = removeExcessFiles(candidateSelection, isUserCompaction, majorCompaction); - - if (candidateSelection.size() == 0 - && candidateFiles.size() >= storeConfigInfo.getBlockingFileCount()) { - candidateSelection = new ArrayList(candidateFiles); - candidateSelection - .subList(0, Math.max(0,candidateSelection.size() - comConf.getMinFilesToCompact())) - .clear(); - } CompactionRequest result = new CompactionRequest(candidateSelection); result.setOffPeak(!candidateSelection.isEmpty() && !majorCompaction && mayUseOffPeak); return result; @@ -261,8 +259,8 @@ public class RatioBasedCompactionPolicy * | | | | | | | | | | | | * | | | | | | | | | | | | */ - ArrayList applyCompactionPolicy( - ArrayList candidates, boolean mayUseOffPeak) throws IOException { + ArrayList applyCompactionPolicy(ArrayList candidates, + boolean mayUseOffPeak, boolean mayBeStuck) throws IOException { if (candidates.isEmpty()) { return candidates; } @@ -298,11 +296,14 @@ public class RatioBasedCompactionPolicy if (start < countOfFiles) { LOG.info("Default compaction algorithm has selected " + (countOfFiles - start) + " files from " + countOfFiles + " candidates"); + } else if (mayBeStuck) { + // We may be stuck. Compact the latest files if we can. + int filesToLeave = candidates.size() - comConf.getMinFilesToCompact(); + if (filesToLeave >= 0) { + start = filesToLeave; + } } - - if (start > 0) { - candidates.subList(0, start).clear(); - } + candidates.subList(0, start).clear(); return candidates; } Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java?rev=1477391&r1=1477390&r2=1477391&view=diff ============================================================================== --- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java (original) +++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java Mon Apr 29 22:36:56 2013 @@ -43,7 +43,7 @@ public class EverythingPolicy extends Ra @Override final ArrayList applyCompactionPolicy(final ArrayList candidates, - final boolean mayUseOffPeak) throws IOException { + final boolean mayUseOffPeak, final boolean mayBeStuck) throws IOException { if (candidates.size() < comConf.getMinFilesToCompact()) { return new ArrayList(0);