hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject hbase git commit: HBASE-14567 Tuneup hanging test TestMobCompactor and TestMobSweeper; ADDENDUM (jingcheng du)
Date Sat, 10 Oct 2015 20:31:31 GMT
Repository: hbase
Updated Branches:
  refs/heads/master e37e72c16 -> 587f5bc11


HBASE-14567 Tuneup hanging test TestMobCompactor and TestMobSweeper; ADDENDUM (jingcheng du)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/587f5bc1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/587f5bc1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/587f5bc1

Branch: refs/heads/master
Commit: 587f5bc11f9d5d37557baf36c7df110af860a95c
Parents: e37e72c
Author: stack <stack@apache.org>
Authored: Sat Oct 10 13:31:17 2015 -0700
Committer: stack <stack@apache.org>
Committed: Sat Oct 10 13:31:17 2015 -0700

----------------------------------------------------------------------
 .../hbase/mob/compactions/TestMobCompactor.java | 484 +++++++------------
 1 file changed, 166 insertions(+), 318 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/587f5bc1/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
index 986649a..6da7655 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java
@@ -68,8 +68,6 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.HFile;
 import org.apache.hadoop.hbase.mob.MobConstants;
 import org.apache.hadoop.hbase.mob.MobUtils;
-import org.apache.hadoop.hbase.mob.compactions.MobCompactor;
-import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactor;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
 import org.apache.hadoop.hbase.regionserver.BloomType;
 import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -81,10 +79,8 @@ import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.Threads;
-import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Assert;
-import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -92,17 +88,16 @@ import org.junit.experimental.categories.Category;
 @Category(LargeTests.class)
 public class TestMobCompactor {
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private Configuration conf = null;
-  private String tableNameAsString;
+  private static Configuration conf = null;
   private TableName tableName;
   private static Connection conn;
   private BufferedMutator bufMut;
-  private Table hTable;
-  private Admin admin;
+  private Table table;
+  private static Admin admin;
   private HTableDescriptor desc;
   private HColumnDescriptor hcd1;
   private HColumnDescriptor hcd2;
-  private FileSystem fs;
+  private static FileSystem fs;
   private static final String family1 = "family1";
   private static final String family2 = "family2";
   private static final String qf1 = "qualifier1";
@@ -127,6 +122,9 @@ public class TestMobCompactor {
     TEST_UTIL.startMiniCluster(1);
     pool = createThreadPool(TEST_UTIL.getConfiguration());
     conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration(), pool);
+    fs = TEST_UTIL.getTestFileSystem();
+    conf = TEST_UTIL.getConfiguration();
+    admin = TEST_UTIL.getHBaseAdmin();
   }
 
   @AfterClass
@@ -136,249 +134,128 @@ public class TestMobCompactor {
     TEST_UTIL.shutdownMiniCluster();
   }
 
-  @Before
-  public void setUp() throws Exception {
-    fs = TEST_UTIL.getTestFileSystem();
-    conf = TEST_UTIL.getConfiguration();
-    long tid = System.currentTimeMillis();
-    tableNameAsString = "testMob" + tid;
+  public void setUp(String tableNameAsString) throws IOException {
     tableName = TableName.valueOf(tableNameAsString);
     hcd1 = new HColumnDescriptor(family1);
     hcd1.setMobEnabled(true);
     hcd1.setMobThreshold(5);
-    hcd1.setMaxVersions(4);
     hcd2 = new HColumnDescriptor(family2);
     hcd2.setMobEnabled(true);
     hcd2.setMobThreshold(5);
-    hcd2.setMaxVersions(4);
     desc = new HTableDescriptor(tableName);
     desc.addFamily(hcd1);
     desc.addFamily(hcd2);
-    admin = TEST_UTIL.getHBaseAdmin();
     admin.createTable(desc, getSplitKeys());
-    hTable = conn.getTable(tableName);
+    table = conn.getTable(tableName);
     bufMut = conn.getBufferedMutator(tableName);
   }
 
-  @After
-  public void tearDown() throws Exception {
-    admin.disableTable(tableName);
-    admin.deleteTable(tableName);
-    admin.close();
-    hTable.close();
-    fs.delete(TEST_UTIL.getDataTestDir(), true);
-  }
-
   @Test(timeout = 300000)
-  public void testCompactionWithoutDelFiles() throws Exception {
+  public void testMinorCompaction() throws Exception {
     resetConf();
+    int mergeSize = 5000;
+    // change the mob compaction merge size
+    conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
+
     // create a table with namespace
     NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create("ns").build();
-    String tableNameAsString = "ns:testCompactionWithoutDelFiles";
+    String tableNameAsString = "ns:testMinorCompaction";
     admin.createNamespace(namespaceDescriptor);
-    Configuration conf = TEST_UTIL.getConfiguration();
-    SecureRandom rng = new SecureRandom();
-    byte[] keyBytes = new byte[AES.KEY_LENGTH];
-    rng.nextBytes(keyBytes);
-    String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
-    Key cfKey = new SecretKeySpec(keyBytes, algorithm);
-    byte[] encryptionKey = EncryptionUtil.wrapKey(conf,
-      conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()),
cfKey);
-    TableName tableName = TableName.valueOf(tableNameAsString);
-    HTableDescriptor desc = new HTableDescriptor(tableName);
-    HColumnDescriptor hcd = new HColumnDescriptor(family1);
-    hcd.setMobEnabled(true);
-    hcd.setMobThreshold(0);
-    hcd.setMaxVersions(4);
-    hcd.setEncryptionType(algorithm);
-    hcd.setEncryptionKey(encryptionKey);
-    HColumnDescriptor hcd2 = new HColumnDescriptor(family2);
-    hcd2.setMobEnabled(true);
-    hcd2.setMobThreshold(0);
-    hcd2.setMaxVersions(4);
-    desc.addFamily(hcd);
-    desc.addFamily(hcd2);
-    admin.createTable(desc, getSplitKeys());
-    Table table = conn.getTable(tableName);
-    BufferedMutator bufMut = conn.getBufferedMutator(tableName);
+    setUp(tableNameAsString);
     int count = 4;
     // generate mob files
     loadData(admin, bufMut, tableName, count, rowNumPerFile);
     int rowNumPerRegion = count * rowNumPerFile;
 
-    assertEquals("Before compaction: mob rows count", regionNum * rowNumPerRegion,
+    assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion,
       countMobRows(table));
-    assertEquals("Before compaction: mob file count", regionNum * count,
+    assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion,
+      countMobCells(table));
+    assertEquals("Before deleting: mob file count", regionNum * count,
       countFiles(tableName, true, family1));
-    assertEquals("Before compaction: del file count", 0, countFiles(tableName, false, family1));
 
-    MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd, pool);
-    compactor.compact();
+    int largeFilesCount = countLargeFiles(mergeSize, tableName, family1);
+    createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1));
 
-    assertEquals("After compaction: mob rows count", regionNum * rowNumPerRegion,
+    assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
       countMobRows(table));
-    assertEquals("After compaction: mob file count", regionNum,
+    assertEquals("Before compaction: mob cells count", regionNum
+      * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
+    assertEquals("Before compaction: family1 mob file count", regionNum * count,
       countFiles(tableName, true, family1));
-    assertEquals("After compaction: del file count", 0, countFiles(tableName, false, family1));
-    Assert.assertTrue(verifyEncryption(tableName, family1));
-    table.close();
-    admin.disableTable(tableName);
-    admin.deleteTable(tableName);
-    admin.deleteNamespace("ns");
-  }
-
-  @Test(timeout = 300000)
-  public void testCompactionWithDelFiles() throws Exception {
-    resetConf();
-    int batchSize = 2;
-    conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, batchSize);
-    int count = 4;
-    // generate mob files
-    loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count*rowNumPerFile;
-
-    assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion,
-        countMobCells(hTable));
-    assertEquals("Before deleting: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before deleting: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-
-    createDelFile();
-
-    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("Before compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("Before compaction: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before compaction: family2 file count", regionNum*count,
-        countFiles(tableName, true, family2));
+    assertEquals("Before compaction: family2 mob file count", regionNum * count,
+      countFiles(tableName, true, family2));
     assertEquals("Before compaction: family1 del file count", regionNum,
-        countFiles(tableName, false, family1));
-    assertEquals("Before compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-
-    // do the mob file compaction
-    MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool);
-    compactor.compact();
-
-    assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("After compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("After compaction: family1 mob file count", regionNum*(count/batchSize),
-        countFiles(tableName, true, family1));
-    assertEquals("After compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("After compaction: family1 del file count", 0,
       countFiles(tableName, false, family1));
-    assertEquals("After compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-    assertRefFileNameEqual(family1);
-  }
-
-  @Test(timeout = 300000)
-  public void testMinorCompactionWithDelFiles() throws Exception {
-    resetConf();
-    int mergeSize = 5000;
-    // change the mob compaction merge size
-    conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
-
-    int count = 4;
-    // generate mob files
-    loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count*rowNumPerFile;
-
-    assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion,
-        countMobCells(hTable));
-    assertEquals("Before deleting: mob file count", regionNum * count,
-      countFiles(tableName, true, family1));
-
-    int largeFilesCount = countLargeFiles(mergeSize, family1);
-    createDelFile();
-
-    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("Before compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("Before compaction: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("Before compaction: family1 del file count", regionNum,
-        countFiles(tableName, false, family1));
     assertEquals("Before compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
+      countFiles(tableName, false, family2));
 
     // do the mob file compaction
     MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool);
     compactor.compact();
 
-    assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("After compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
+    assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
+      countMobRows(table));
+    assertEquals("After compaction: mob cells count", regionNum
+      * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
     // After the compaction, the files smaller than the mob compaction merge size
     // is merge to one file
     assertEquals("After compaction: family1 mob file count", largeFilesCount + regionNum,
-        countFiles(tableName, true, family1));
-    assertEquals("After compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
+      countFiles(tableName, true, family1));
+    assertEquals("After compaction: family2 mob file count", regionNum * count,
+      countFiles(tableName, true, family2));
     assertEquals("After compaction: family1 del file count", regionNum,
-        countFiles(tableName, false, family1));
+      countFiles(tableName, false, family1));
     assertEquals("After compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
+      countFiles(tableName, false, family2));
   }
 
   @Test(timeout = 300000)
   public void testCompactionWithHFileLink() throws IOException, InterruptedException {
     resetConf();
+    String tableNameAsString = "testCompactionWithHFileLink";
+    setUp(tableNameAsString);
     int count = 4;
     // generate mob files
     loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count*rowNumPerFile;
+    int rowNumPerRegion = count * rowNumPerFile;
 
     long tid = System.currentTimeMillis();
     byte[] snapshotName1 = Bytes.toBytes("snaptb-" + tid);
     // take a snapshot
     admin.snapshot(snapshotName1, tableName);
 
-    createDelFile();
+    createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1));
 
-    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("Before compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("Before compaction: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
+    assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
+      countMobRows(table));
+    assertEquals("Before compaction: mob cells count", regionNum
+      * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
+    assertEquals("Before compaction: family1 mob file count", regionNum * count,
+      countFiles(tableName, true, family1));
+    assertEquals("Before compaction: family2 mob file count", regionNum * count,
+      countFiles(tableName, true, family2));
     assertEquals("Before compaction: family1 del file count", regionNum,
-        countFiles(tableName, false, family1));
+      countFiles(tableName, false, family1));
     assertEquals("Before compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
+      countFiles(tableName, false, family2));
 
     // do the mob compaction
     MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool);
     compactor.compact();
 
-    assertEquals("After first compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("After first compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
+    assertEquals("After first compaction: mob rows count", regionNum
+      * (rowNumPerRegion - delRowNum), countMobRows(table));
+    assertEquals("After first compaction: mob cells count", regionNum
+      * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
     assertEquals("After first compaction: family1 mob file count", regionNum,
-        countFiles(tableName, true, family1));
-    assertEquals("After first compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
+      countFiles(tableName, true, family1));
+    assertEquals("After first compaction: family2 mob file count", regionNum * count,
+      countFiles(tableName, true, family2));
     assertEquals("After first compaction: family1 del file count", 0,
       countFiles(tableName, false, family1));
     assertEquals("After first compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
+      countFiles(tableName, false, family2));
     assertEquals("After first compaction: family1 hfilelink count", 0, countHFileLinks(family1));
     assertEquals("After first compaction: family2 hfilelink count", 0, countHFileLinks(family2));
 
@@ -387,33 +264,32 @@ public class TestMobCompactor {
     admin.restoreSnapshot(snapshotName1);
     admin.enableTable(tableName);
 
-    assertEquals("After restoring snapshot: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("After restoring snapshot: mob cells count",
-        regionNum*cellNumPerRow*rowNumPerRegion, countMobCells(hTable));
-    assertEquals("After restoring snapshot: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("After restoring snapshot: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
+    assertEquals("After restoring snapshot: mob rows count", regionNum * rowNumPerRegion,
+      countMobRows(table));
+    assertEquals("After restoring snapshot: mob cells count", regionNum * cellNumPerRow
+      * rowNumPerRegion, countMobCells(table));
+    assertEquals("After restoring snapshot: family1 mob file count", regionNum * count,
+      countFiles(tableName, true, family1));
+    assertEquals("After restoring snapshot: family2 mob file count", regionNum * count,
+      countFiles(tableName, true, family2));
     assertEquals("After restoring snapshot: family1 del file count", 0,
-        countFiles(tableName, false, family1));
+      countFiles(tableName, false, family1));
     assertEquals("After restoring snapshot: family2 del file count", 0,
-        countFiles(tableName, false, family2));
-    assertEquals("After restoring snapshot: family1 hfilelink count", regionNum*count,
-        countHFileLinks(family1));
-    assertEquals("After restoring snapshot: family2 hfilelink count", 0,
-        countHFileLinks(family2));
+      countFiles(tableName, false, family2));
+    assertEquals("After restoring snapshot: family1 hfilelink count", regionNum * count,
+      countHFileLinks(family1));
+    assertEquals("After restoring snapshot: family2 hfilelink count", 0, countHFileLinks(family2));
 
     compactor.compact();
 
-    assertEquals("After second compaction: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("After second compaction: mob cells count",
-        regionNum*cellNumPerRow*rowNumPerRegion, countMobCells(hTable));
+    assertEquals("After second compaction: mob rows count", regionNum * rowNumPerRegion,
+      countMobRows(table));
+    assertEquals("After second compaction: mob cells count", regionNum * cellNumPerRow
+      * rowNumPerRegion, countMobCells(table));
     assertEquals("After second compaction: family1 mob file count", regionNum,
-        countFiles(tableName, true, family1));
-    assertEquals("After second compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
+      countFiles(tableName, true, family1));
+    assertEquals("After second compaction: family2 mob file count", regionNum * count,
+      countFiles(tableName, true, family2));
     assertEquals("After second compaction: family1 del file count", 0,
       countFiles(tableName, false, family1));
     assertEquals("After second compaction: family2 del file count", 0,
@@ -424,114 +300,85 @@ public class TestMobCompactor {
   }
 
   @Test(timeout = 300000)
-  public void testCompactionFromAdmin() throws Exception {
+  public void testMajorCompactionFromAdmin() throws Exception {
     resetConf();
     int mergeSize = 5000;
     // change the mob compaction merge size
     conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
+    String tableNameAsString = "testMajorCompactionFromAdmin";
+    SecureRandom rng = new SecureRandom();
+    byte[] keyBytes = new byte[AES.KEY_LENGTH];
+    rng.nextBytes(keyBytes);
+    String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
+    Key cfKey = new SecretKeySpec(keyBytes, algorithm);
+    byte[] encryptionKey = EncryptionUtil.wrapKey(conf,
+      conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()),
cfKey);
+    TableName tableName = TableName.valueOf(tableNameAsString);
+    HTableDescriptor desc = new HTableDescriptor(tableName);
+    HColumnDescriptor hcd1 = new HColumnDescriptor(family1);
+    hcd1.setMobEnabled(true);
+    hcd1.setMobThreshold(0);
+    hcd1.setEncryptionType(algorithm);
+    hcd1.setEncryptionKey(encryptionKey);
+    HColumnDescriptor hcd2 = new HColumnDescriptor(family2);
+    hcd2.setMobEnabled(true);
+    hcd2.setMobThreshold(0);
+    desc.addFamily(hcd1);
+    desc.addFamily(hcd2);
+    admin.createTable(desc, getSplitKeys());
+    Table table = conn.getTable(tableName);
+    BufferedMutator bufMut = conn.getBufferedMutator(tableName);
     int count = 4;
     // generate mob files
     loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count*rowNumPerFile;
-
-    assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion,
-        countMobCells(hTable));
-    assertEquals("Before deleting: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before deleting: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-
-    createDelFile();
-
-    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("Before compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("Before compaction: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before compaction: family2 file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("Before compaction: family1 del file count", regionNum,
-        countFiles(tableName, false, family1));
-    assertEquals("Before compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
+    int rowNumPerRegion = count * rowNumPerFile;
 
-    int largeFilesCount = countLargeFiles(5000, family1);
-    // do the mob compaction
-    admin.compact(tableName, hcd1.getName(), Admin.CompactType.MOB);
+    assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion,
+      countMobRows(table));
+    assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion,
+      countMobCells(table));
+    assertEquals("Before deleting: mob file count", regionNum * count,
+      countFiles(tableName, true, family1));
 
-    waitUntilMobCompactionFinished(tableName);
-    assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
-      countMobRows(hTable));
-    assertEquals("After compaction: mob cells count", regionNum
-      * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(hTable));
-    assertEquals("After compaction: family1 mob file count", regionNum + largeFilesCount,
+    createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1));
+
+    assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
+      countMobRows(table));
+    assertEquals("Before compaction: mob cells count", regionNum
+      * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
+    assertEquals("Before compaction: family1 mob file count", regionNum * count,
       countFiles(tableName, true, family1));
-    assertEquals("After compaction: family2 mob file count", regionNum * count,
+    assertEquals("Before compaction: family2 mob file count", regionNum * count,
       countFiles(tableName, true, family2));
-    assertEquals("After compaction: family1 del file count", regionNum,
-      countFiles(tableName, false, family1));
-    assertEquals("After compaction: family2 del file count", regionNum,
-      countFiles(tableName, false, family2));
-    assertRefFileNameEqual(family1);
-  }
-
-  @Test(timeout = 300000)
-  public void testMajorCompactionFromAdmin() throws Exception {
-    resetConf();
-    int mergeSize = 5000;
-    // change the mob compaction merge size
-    conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
-    int count = 4;
-    // generate mob files
-    loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count*rowNumPerFile;
-
-    assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion,
-        countMobCells(hTable));
-    assertEquals("Before deleting: mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-
-    createDelFile();
-
-    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("Before compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("Before compaction: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
     assertEquals("Before compaction: family1 del file count", regionNum,
-        countFiles(tableName, false, family1));
+      countFiles(tableName, false, family1));
     assertEquals("Before compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
+      countFiles(tableName, false, family2));
 
     // do the major mob compaction, it will force all files to compaction
     admin.majorCompact(tableName, hcd1.getName(), Admin.CompactType.MOB);
 
     waitUntilMobCompactionFinished(tableName);
-    assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("After compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
+    assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
+      countMobRows(table));
+    assertEquals("After compaction: mob cells count", regionNum
+      * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
     assertEquals("After compaction: family1 mob file count", regionNum,
-        countFiles(tableName, true, family1));
-    assertEquals("After compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
+      countFiles(tableName, true, family1));
+    assertEquals("After compaction: family2 mob file count", regionNum * count,
+      countFiles(tableName, true, family2));
     assertEquals("After compaction: family1 del file count", 0,
-        countFiles(tableName, false, family1));
+      countFiles(tableName, false, family1));
     assertEquals("After compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
+      countFiles(tableName, false, family2));
+    Assert.assertTrue(verifyEncryption(tableName, family1));
+    table.close();
   }
 
   @Test(timeout = 300000)
   public void testScannerOnBulkLoadRefHFiles() throws Exception {
     resetConf();
+    setUp("testScannerOnBulkLoadRefHFiles");
     long ts = EnvironmentEdgeManager.currentTime();
     byte[] key0 = Bytes.toBytes("k0");
     byte[] key1 = Bytes.toBytes("k1");
@@ -548,7 +395,7 @@ public class TestMobCompactor {
     loadData(admin, bufMut, tableName, new Put[] { put0, put1 });
     // read the latest cell of key0.
     Get get = new Get(key0);
-    Result result = hTable.get(get);
+    Result result = table.get(get);
     Cell cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1));
     assertEquals("Before compaction: mob value of k0", newValue0,
       Bytes.toString(CellUtil.cloneValue(cell)));
@@ -556,18 +403,18 @@ public class TestMobCompactor {
     waitUntilMobCompactionFinished(tableName);
     // read the latest cell of key0, the cell seqId in bulk loaded file is not reset in the
     // scanner. The cell that has "new" value is still visible.
-    result = hTable.get(get);
+    result = table.get(get);
     cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1));
     assertEquals("After compaction: mob value of k0", newValue0,
       Bytes.toString(CellUtil.cloneValue(cell)));
     // read the ref cell, not read further to the mob cell.
     get = new Get(key1);
     get.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(true));
-    result = hTable.get(get);
+    result = table.get(get);
     cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1));
     // the ref name is the new file
-    Path mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(),
-      tableName), hcd1.getNameAsString());
+    Path mobFamilyPath = new Path(
+      MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(), tableName), hcd1.getNameAsString());
     List<Path> paths = new ArrayList<Path>();
     if (fs.exists(mobFamilyPath)) {
       FileStatus[] files = fs.listStatus(mobFamilyPath);
@@ -585,6 +432,7 @@ public class TestMobCompactor {
   @Test(timeout = 300000)
   public void testScannerAfterCompactions() throws Exception {
     resetConf();
+    setUp("testScannerAfterCompactions");
     long ts = EnvironmentEdgeManager.currentTime();
     byte[] key0 = Bytes.toBytes("k0");
     byte[] key1 = Bytes.toBytes("k1");
@@ -605,7 +453,7 @@ public class TestMobCompactor {
     waitUntilMobCompactionFinished(tableName);
     // read the latest cell of key1.
     Get get = new Get(key1);
-    Result result = hTable.get(get);
+    Result result = table.get(get);
     Cell cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1));
     assertEquals("After compaction: mob value", "new", Bytes.toString(CellUtil.cloneValue(cell)));
   }
@@ -748,19 +596,19 @@ public class TestMobCompactor {
   /**
    * Gets the number of files.
    * @param size the size of the file
+   * @param tableName the current table name
    * @param familyName the family name
    * @return the number of files large than the size
    */
-  private int countLargeFiles(int size, String familyName) throws IOException {
-    Path mobDirPath = MobUtils.getMobFamilyPath(
-        MobUtils.getMobRegionPath(conf, tableName), familyName);
+  private int countLargeFiles(int size, TableName tableName, String familyName) throws IOException
{
+    Path mobDirPath = MobUtils.getMobFamilyPath(MobUtils.getMobRegionPath(conf, tableName),
+      familyName);
     int count = 0;
     if (fs.exists(mobDirPath)) {
       FileStatus[] files = fs.listStatus(mobDirPath);
       for (FileStatus file : files) {
         // ignore the del files in the mob path
-        if ((!StoreFileInfo.isDelFile(file.getPath()))
-            && (file.getLen() > size)) {
+        if ((!StoreFileInfo.isDelFile(file.getPath())) && (file.getLen() > size))
{
           count++;
         }
       }
@@ -776,9 +624,9 @@ public class TestMobCompactor {
     if (fileNum <= 0) {
       throw new IllegalArgumentException();
     }
-    for (byte k0 : KEYS) {
-      byte[] k = new byte[] { k0 };
-      for (int i = 0; i < fileNum * rowNumPerFile; i++) {
+    for (int i = 0; i < fileNum * rowNumPerFile; i++) {
+      for (byte k0 : KEYS) {
+        byte[] k = new byte[] { k0 };
         byte[] key = Bytes.add(k, Bytes.toBytes(i));
         byte[] mobVal = makeDummyData(10 * (i + 1));
         Put put = new Put(key);
@@ -787,10 +635,10 @@ public class TestMobCompactor {
         put.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf2), mobVal);
         put.addColumn(Bytes.toBytes(family2), Bytes.toBytes(qf1), mobVal);
         table.mutate(put);
-        if ((i + 1) % rowNumPerFile == 0) {
-          table.flush();
-          admin.flush(tableName);
-        }
+      }
+      if ((i + 1) % rowNumPerFile == 0) {
+        table.flush();
+        admin.flush(tableName);
       }
     }
   }
@@ -805,30 +653,30 @@ public class TestMobCompactor {
   /**
    * delete the row, family and cell to create the del file
    */
-  private void createDelFile() throws IOException, InterruptedException {
+  private void createDelFile(Table table, TableName tableName, byte[] family, byte[] qf)
+    throws IOException, InterruptedException {
     for (byte k0 : KEYS) {
       byte[] k = new byte[] { k0 };
       // delete a family
       byte[] key1 = Bytes.add(k, Bytes.toBytes(0));
       Delete delete1 = new Delete(key1);
-      delete1.addFamily(Bytes.toBytes(family1));
-      hTable.delete(delete1);
+      delete1.addFamily(family);
+      table.delete(delete1);
       // delete one row
       byte[] key2 = Bytes.add(k, Bytes.toBytes(2));
       Delete delete2 = new Delete(key2);
-      hTable.delete(delete2);
+      table.delete(delete2);
       // delete one cell
       byte[] key3 = Bytes.add(k, Bytes.toBytes(4));
       Delete delete3 = new Delete(key3);
-      delete3.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1));
-      hTable.delete(delete3);
-      admin.flush(tableName);
-      List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(
-          Bytes.toBytes(tableNameAsString));
-      for (HRegion region : regions) {
-        region.waitForFlushesAndCompactions();
-        region.compact(true);
-      }
+      delete3.addColumn(family, qf);
+      table.delete(delete3);
+    }
+    admin.flush(tableName);
+    List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(tableName);
+    for (HRegion region : regions) {
+      region.waitForFlushesAndCompactions();
+      region.compact(true);
     }
   }
   /**
@@ -880,7 +728,7 @@ public class TestMobCompactor {
     scan.addFamily(Bytes.toBytes(familyName));
     // Do not retrieve the mob data when scanning
     scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
-    ResultScanner results = hTable.getScanner(scan);
+    ResultScanner results = table.getScanner(scan);
     Path mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(),
         tableName), familyName);
     List<Path> actualFilePaths = new ArrayList<>();


Mime
View raw message