hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject svn commit: r1526320 - in /hbase/trunk/hbase-server/src: main/java/org/apache/hadoop/hbase/regionserver/compactions/ test/java/org/apache/hadoop/hbase/ test/java/org/apache/hadoop/hbase/coprocessor/ test/java/org/apache/hadoop/hbase/master/cleaner/
Date Wed, 25 Sep 2013 22:17:35 GMT
Author: sershe
Date: Wed Sep 25 22:17:35 2013
New Revision: 1526320

URL: http://svn.apache.org/r1526320
Log:
HBASE-8870 Store.needsCompaction() should include minFilesToCompact (Liang Xie)

Modified:
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java?rev=1526320&r1=1526319&r2=1526320&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
(original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
Wed Sep 25 22:17:35 2013
@@ -392,6 +392,6 @@ public class RatioBasedCompactionPolicy 
   public boolean needsCompaction(final Collection<StoreFile> storeFiles,
       final List<StoreFile> filesCompacting) {
     int numCandidates = storeFiles.size() - filesCompacting.size();
-    return numCandidates > comConf.getMinFilesToCompact();
+    return numCandidates >= comConf.getMinFilesToCompact();
   }
 }

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1526320&r1=1526319&r2=1526320&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
Wed Sep 25 22:17:35 2013
@@ -1167,28 +1167,40 @@ public class HBaseTestingUtility extends
 
   /**
    * Create a table.
-   * @param tableName
+   * @param htd
    * @param families
    * @param c Configuration to use
    * @return An HTable instance for the created table.
    * @throws IOException
    */
-  public HTable createTable(TableName tableName, byte[][] families,
-      final Configuration c)
+  public HTable createTable(HTableDescriptor htd, byte[][] families, Configuration c)
   throws IOException {
-    HTableDescriptor desc = new HTableDescriptor(tableName);
     for(byte[] family : families) {
       HColumnDescriptor hcd = new HColumnDescriptor(family);
       // Disable blooms (they are on by default as of 0.95) but we disable them here because
       // tests have hard coded counts of what to expect in block cache, etc., and blooms
being
       // on is interfering.
       hcd.setBloomFilterType(BloomType.NONE);
-      desc.addFamily(hcd);
+      htd.addFamily(hcd);
     }
-    getHBaseAdmin().createTable(desc);
+    getHBaseAdmin().createTable(htd);
     // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they
are assigned
-    waitUntilAllRegionsAssigned(tableName);
-    return new HTable(c, tableName);
+    waitUntilAllRegionsAssigned(htd.getTableName());
+    return new HTable(c, htd.getTableName());
+  }
+
+  /**
+   * Create a table.
+   * @param tableName
+   * @param families
+   * @param c Configuration to use
+   * @return An HTable instance for the created table.
+   * @throws IOException
+   */
+  public HTable createTable(TableName tableName, byte[][] families,
+      final Configuration c)
+  throws IOException {
+    return createTable(new HTableDescriptor(tableName), families, c);
   }
 
   /**

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java?rev=1526320&r1=1526319&r2=1526320&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
Wed Sep 25 22:17:35 2013
@@ -55,9 +55,7 @@ import org.apache.hadoop.hbase.regionser
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.ScanType;
 import org.apache.hadoop.hbase.regionserver.Store;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.StoreScanner;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -249,14 +247,8 @@ public class TestRegionObserverScannerOp
     admin.flush(region.getRegionName());
 
     // run a compaction, which normally would should get rid of the data
-    Store s = region.getStores().get(A);
-    CountDownLatch latch = new CountDownLatch(1);
-    WaitableCompactionRequest request = new WaitableCompactionRequest(s.getStorefiles(),
latch);
-    rs.compactSplitThread.requestCompaction(region, s,
-      "compact for testRegionObserverCompactionTimeStacking", Store.PRIORITY_USER, request);
-    // wait for the compaction to complete
-    latch.await();
-
+    // wait for the compaction checker to complete
+    Thread.sleep(1000);
     // check both rows to ensure that they aren't there
     Get get = new Get(ROW);
     Result r = table.get(get);
@@ -273,26 +265,4 @@ public class TestRegionObserverScannerOp
     table.close();
     UTIL.shutdownMiniCluster();
   }
-
-  /**
-   * A simple compaction on which you can wait for the passed in latch until the compaction
finishes
-   * (either successfully or if it failed).
-   */
-  public static class WaitableCompactionRequest extends CompactionRequest {
-    private CountDownLatch done;
-
-    /**
-     * Constructor for a custom compaction. Uses the setXXX methods to update the state of
the
-     * compaction before being used.
-     */
-    public WaitableCompactionRequest(Collection<StoreFile> files, CountDownLatch finished)
{
-      super(files);
-      this.done = finished;
-    }
-
-    @Override
-    public void afterExecute() {
-      this.done.countDown();
-    }
-  }
 }
\ No newline at end of file

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java?rev=1526320&r1=1526319&r2=1526320&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
(original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
Wed Sep 25 22:17:35 2013
@@ -33,6 +33,7 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
@@ -280,11 +281,19 @@ public class TestSnapshotFromMaster {
     HBaseAdmin admin = UTIL.getHBaseAdmin();
     // make sure we don't fail on listing snapshots
     SnapshotTestingUtils.assertNoSnapshots(admin);
+
+    // recreate test table with disabled compactions; otherwise compaction may happen before
+    // snapshot, the call after snapshot will be a no-op and checks will fail
+    UTIL.deleteTable(TABLE_NAME);
+    HTableDescriptor htd = new HTableDescriptor(TABLE_NAME);
+    htd.setCompactionEnabled(false);
+    UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
     // load the table (creates 4 hfiles)
     UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE_NAME), TEST_FAM);
 
     // disable the table so we can take a snapshot
     admin.disableTable(TABLE_NAME);
+    htd.setCompactionEnabled(true);
 
     // take a snapshot of the table
     String snapshotName = "snapshot";
@@ -298,6 +307,9 @@ public class TestSnapshotFromMaster {
     // ensure we only have one snapshot
     SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshotNameBytes, TABLE_NAME);
 
+    // enable compactions now
+    admin.modifyTable(TABLE_NAME, htd);
+
     // renable the table so we can compact the regions
     admin.enableTable(TABLE_NAME);
 



Mime
View raw message