hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e...@apache.org
Subject [1/3] hbase git commit: HBASE-16301 Trigger flush without waiting when compaction is disabled on a table (huaxiang sun)
Date Wed, 03 Aug 2016 18:02:26 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1 b06119ede -> 12549de43
  refs/heads/branch-1.3 b621184f8 -> 42de52860
  refs/heads/master e63fb6c02 -> bcf2738a0


HBASE-16301 Trigger flush without waiting when compaction is disabled on a table (huaxiang
sun)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bcf2738a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bcf2738a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bcf2738a

Branch: refs/heads/master
Commit: bcf2738a0135c4e18a6c0975964cb6d2375a2806
Parents: e63fb6c
Author: Enis Soztutar <enis@apache.org>
Authored: Tue Aug 2 18:46:07 2016 -0700
Committer: Enis Soztutar <enis@apache.org>
Committed: Tue Aug 2 18:46:07 2016 -0700

----------------------------------------------------------------------
 .../hbase/regionserver/MemStoreFlusher.java     |  6 ++
 .../regionserver/TestCompactSplitThread.java    | 84 ++++++++++++++++++--
 2 files changed, 85 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/bcf2738a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
index a69d8c0..09b3738 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
@@ -551,6 +551,12 @@ class MemStoreFlusher implements FlushRequester {
   }
 
   private boolean isTooManyStoreFiles(Region region) {
+
+    // When compaction is disabled, the region is flushable
+    if (!region.getTableDesc().isCompactionEnabled()) {
+      return false;
+    }
+
     for (Store store : region.getStores()) {
       if (store.hasTooManyStoreFiles()) {
         return true;

http://git-wip-us.apache.org/repos/asf/hbase/blob/bcf2738a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSplitThread.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSplitThread.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSplitThread.java
index 3457883..6c851de 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSplitThread.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSplitThread.java
@@ -17,15 +17,24 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
+import java.util.Collection;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
+
 import org.junit.experimental.categories.Category;
 
 import static org.junit.Assert.assertEquals;
@@ -36,15 +45,61 @@ public class TestCompactSplitThread {
   private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private final TableName tableName = TableName.valueOf(getClass().getSimpleName());
   private final byte[] family = Bytes.toBytes("f");
+  private static final int NUM_RS = 1;
+  private static final int blockingStoreFiles = 3;
+  private static Path rootDir;
+  private static FileSystem fs;
 
-  @Test
-  public void testThreadPoolSizeTuning() throws Exception {
-    Configuration conf = TEST_UTIL.getConfiguration();
+
+
+  /**
+   * Setup the config for the cluster
+   */
+  @BeforeClass
+  public static void setupCluster() throws Exception {
+    setupConf(TEST_UTIL.getConfiguration());
+    TEST_UTIL.startMiniCluster(NUM_RS);
+    fs = TEST_UTIL.getDFSCluster().getFileSystem();
+    rootDir = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
+
+  }
+
+  private static void setupConf(Configuration conf) {
+    // disable the ui
+    conf.setInt("hbase.regionsever.info.port", -1);
+    // so make sure we get a compaction when doing a load, but keep around some
+    // files in the store
+    conf.setInt("hbase.hstore.compaction.min", 2);
+    conf.setInt("hbase.hstore.compactionThreshold", 5);
+    // change the flush size to a small amount, regulating number of store files
+    conf.setInt("hbase.hregion.memstore.flush.size", 25000);
+
+    // block writes if we get to blockingStoreFiles store files
+    conf.setInt("hbase.hstore.blockingStoreFiles", blockingStoreFiles);
+    // Ensure no extra cleaners on by default (e.g. TimeToLiveHFileCleaner)
     conf.setInt(CompactSplitThread.LARGE_COMPACTION_THREADS, 3);
     conf.setInt(CompactSplitThread.SMALL_COMPACTION_THREADS, 4);
     conf.setInt(CompactSplitThread.SPLIT_THREADS, 5);
     conf.setInt(CompactSplitThread.MERGE_THREADS, 6);
-    TEST_UTIL.startMiniCluster(1);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    TEST_UTIL.deleteTable(tableName);
+  }
+
+  @AfterClass
+  public static void cleanupTest() throws Exception {
+    try {
+      TEST_UTIL.shutdownMiniCluster();
+    } catch (Exception e) {
+      // NOOP;
+    }
+  }
+
+  @Test
+  public void testThreadPoolSizeTuning() throws Exception {
+    Configuration conf = TEST_UTIL.getConfiguration();
     Connection conn = ConnectionFactory.createConnection(conf);
     try {
       HTableDescriptor htd = new HTableDescriptor(tableName);
@@ -95,7 +150,26 @@ public class TestCompactSplitThread {
       assertEquals(5, regionServer.compactSplitThread.getMergeThreadNum());
     } finally {
       conn.close();
-      TEST_UTIL.shutdownMiniCluster();
     }
   }
+
+  @Test(timeout = 60000)
+  public void testFlushWithTableCompactionDisabled() throws Exception {
+    Admin admin = TEST_UTIL.getHBaseAdmin();
+
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    htd.setCompactionEnabled(false);
+    TEST_UTIL.createTable(htd, new byte[][] { family }, null);
+
+    // load the table
+    for (int i = 0; i < blockingStoreFiles + 1; i ++) {
+      TEST_UTIL.loadTable(TEST_UTIL.getConnection().getTable(tableName), family);
+      TEST_UTIL.flush(tableName);
+    }
+
+    // Make sure that store file number is greater than blockingStoreFiles + 1
+    Path tableDir = FSUtils.getTableDir(rootDir, tableName);
+    Collection<String> hfiles =  SnapshotTestingUtils.listHFileNames(fs, tableDir);
+    assert(hfiles.size() > blockingStoreFiles + 1);
+  }
 }


Mime
View raw message