hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jmhs...@apache.org
Subject [24/50] [abbrv] hbase git commit: HBASE-13763 Handle the rename, annotation and typo stuff in MOB. (Jingcheng)
Date Wed, 22 Jul 2015 19:52:42 GMT
http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestMobFileCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestMobFileCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestMobFileCompactor.java
deleted file mode 100644
index abdfb94..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestMobFileCompactor.java
+++ /dev/null
@@ -1,922 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.mob.filecompactions;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.security.Key;
-import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.RejectedExecutionHandler;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import javax.crypto.spec.SecretKeySpec;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.BufferedMutator;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.io.HFileLink;
-import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
-import org.apache.hadoop.hbase.io.crypto.aes.AES;
-import org.apache.hadoop.hbase.io.hfile.CacheConfig;
-import org.apache.hadoop.hbase.io.hfile.HFile;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobUtils;
-import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
-import org.apache.hadoop.hbase.regionserver.BloomType;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
-import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
-import org.apache.hadoop.hbase.security.EncryptionUtil;
-import org.apache.hadoop.hbase.security.User;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.Threads;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(LargeTests.class)
-public class TestMobFileCompactor {
-  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private Configuration conf = null;
-  private String tableNameAsString;
-  private TableName tableName;
-  private static Connection conn;
-  private BufferedMutator bufMut;
-  private Table hTable;
-  private Admin admin;
-  private HTableDescriptor desc;
-  private HColumnDescriptor hcd1;
-  private HColumnDescriptor hcd2;
-  private FileSystem fs;
-  private static final String family1 = "family1";
-  private static final String family2 = "family2";
-  private static final String qf1 = "qualifier1";
-  private static final String qf2 = "qualifier2";
-  private static byte[] KEYS = Bytes.toBytes("012");
-  private static int regionNum = KEYS.length;
-  private static int delRowNum = 1;
-  private static int delCellNum = 6;
-  private static int cellNumPerRow = 3;
-  private static int rowNumPerFile = 2;
-  private static ExecutorService pool;
-
-  @BeforeClass
-  public static void setUpBeforeClass() throws Exception {
-    TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
-    TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
-    TEST_UTIL.getConfiguration()
-      .setLong(MobConstants.MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD, 5000);
-    TEST_UTIL.getConfiguration().set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY,
-      KeyProviderForTesting.class.getName());
-    TEST_UTIL.getConfiguration().set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
-    TEST_UTIL.startMiniCluster(1);
-    pool = createThreadPool(TEST_UTIL.getConfiguration());
-    conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration(), pool);
-  }
-
-  @AfterClass
-  public static void tearDownAfterClass() throws Exception {
-    pool.shutdown();
-    conn.close();
-    TEST_UTIL.shutdownMiniCluster();
-  }
-
-  @Before
-  public void setUp() throws Exception {
-    fs = TEST_UTIL.getTestFileSystem();
-    conf = TEST_UTIL.getConfiguration();
-    long tid = System.currentTimeMillis();
-    tableNameAsString = "testMob" + tid;
-    tableName = TableName.valueOf(tableNameAsString);
-    hcd1 = new HColumnDescriptor(family1);
-    hcd1.setMobEnabled(true);
-    hcd1.setMobThreshold(0L);
-    hcd1.setMaxVersions(4);
-    hcd2 = new HColumnDescriptor(family2);
-    hcd2.setMobEnabled(true);
-    hcd2.setMobThreshold(0L);
-    hcd2.setMaxVersions(4);
-    desc = new HTableDescriptor(tableName);
-    desc.addFamily(hcd1);
-    desc.addFamily(hcd2);
-    admin = TEST_UTIL.getHBaseAdmin();
-    admin.createTable(desc, getSplitKeys());
-    hTable = conn.getTable(tableName);
-    bufMut = conn.getBufferedMutator(tableName);
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    admin.disableTable(tableName);
-    admin.deleteTable(tableName);
-    admin.close();
-    hTable.close();
-    fs.delete(TEST_UTIL.getDataTestDir(), true);
-  }
-
-  @Test
-  public void testCompactionWithoutDelFilesWithNamespace() throws Exception {
-    resetConf();
-    // create a table with namespace
-    NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create("ns").build();
-    String tableNameAsString = "ns:testCompactionWithoutDelFilesWithNamespace";
-    admin.createNamespace(namespaceDescriptor);
-    TableName tableName = TableName.valueOf(tableNameAsString);
-    HColumnDescriptor hcd1 = new HColumnDescriptor(family1);
-    hcd1.setMobEnabled(true);
-    hcd1.setMobThreshold(0L);
-    hcd1.setMaxVersions(4);
-    HColumnDescriptor hcd2 = new HColumnDescriptor(family2);
-    hcd2.setMobEnabled(true);
-    hcd2.setMobThreshold(0L);
-    hcd2.setMaxVersions(4);
-    HTableDescriptor desc = new HTableDescriptor(tableName);
-    desc.addFamily(hcd1);
-    desc.addFamily(hcd2);
-    admin.createTable(desc, getSplitKeys());
-    BufferedMutator bufMut= conn.getBufferedMutator(tableName);
-    Table table = conn.getTable(tableName);
-
-    int count = 4;
-    // generate mob files
-    loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count * rowNumPerFile;
-
-    assertEquals("Before compaction: mob rows count", regionNum * rowNumPerRegion,
-      countMobRows(table));
-    assertEquals("Before compaction: mob file count", regionNum * count,
-      countFiles(tableName, true, family1));
-    assertEquals("Before compaction: del file count", 0, countFiles(tableName, false, family1));
-
-    MobFileCompactor compactor = new PartitionedMobFileCompactor(conf, fs, tableName, hcd1, pool);
-    compactor.compact();
-
-    assertEquals("After compaction: mob rows count", regionNum * rowNumPerRegion,
-      countMobRows(table));
-    assertEquals("After compaction: mob file count", regionNum,
-      countFiles(tableName, true, family1));
-    assertEquals("After compaction: del file count", 0, countFiles(tableName, false, family1));
-
-    table.close();
-    admin.disableTable(tableName);
-    admin.deleteTable(tableName);
-    admin.deleteNamespace("ns");
-  }
-
-  @Test
-  public void testCompactionWithoutDelFiles() throws Exception {
-    resetConf();
-    int count = 4;
-    // generate mob files
-    loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count*rowNumPerFile;
-
-    assertEquals("Before compaction: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("Before compaction: mob file count", regionNum * count,
-      countFiles(tableName, true, family1));
-    assertEquals("Before compaction: del file count", 0, countFiles(tableName, false, family1));
-
-    MobFileCompactor compactor = new PartitionedMobFileCompactor(conf, fs, tableName, hcd1, pool);
-    compactor.compact();
-
-    assertEquals("After compaction: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("After compaction: mob file count", regionNum,
-      countFiles(tableName, true, family1));
-    assertEquals("After compaction: del file count", 0, countFiles(tableName, false, family1));
-  }
-
-  @Test
-  public void testCompactionWithoutDelFilesAndWithEncryption() throws Exception {
-    resetConf();
-    Configuration conf = TEST_UTIL.getConfiguration();
-    SecureRandom rng = new SecureRandom();
-    byte[] keyBytes = new byte[AES.KEY_LENGTH];
-    rng.nextBytes(keyBytes);
-    String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
-    Key cfKey = new SecretKeySpec(keyBytes, algorithm);
-    byte[] encryptionKey = EncryptionUtil.wrapKey(conf,
-      conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey);
-    String tableNameAsString = "testCompactionWithoutDelFilesAndWithEncryption";
-    TableName tableName = TableName.valueOf(tableNameAsString);
-    HTableDescriptor desc = new HTableDescriptor(tableName);
-    HColumnDescriptor hcd = new HColumnDescriptor(family1);
-    hcd.setMobEnabled(true);
-    hcd.setMobThreshold(0);
-    hcd.setMaxVersions(4);
-    hcd.setEncryptionType(algorithm);
-    hcd.setEncryptionKey(encryptionKey);
-    HColumnDescriptor hcd2 = new HColumnDescriptor(family2);
-    hcd2.setMobEnabled(true);
-    hcd2.setMobThreshold(0);
-    hcd2.setMaxVersions(4);
-    desc.addFamily(hcd);
-    desc.addFamily(hcd2);
-    admin.createTable(desc, getSplitKeys());
-    Table hTable = conn.getTable(tableName);
-    BufferedMutator bufMut = conn.getBufferedMutator(tableName);
-    int count = 4;
-    // generate mob files
-    loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count*rowNumPerFile;
-
-    assertEquals("Before compaction: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("Before compaction: mob file count", regionNum * count,
-      countFiles(tableName, true, family1));
-    assertEquals("Before compaction: del file count", 0, countFiles(tableName, false, family1));
-
-    MobFileCompactor compactor = new PartitionedMobFileCompactor(conf, fs, tableName, hcd, pool);
-    compactor.compact();
-
-    assertEquals("After compaction: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("After compaction: mob file count", regionNum,
-      countFiles(tableName, true, family1));
-    assertEquals("After compaction: del file count", 0, countFiles(tableName, false, family1));
-    Assert.assertTrue(verifyEncryption(tableName, family1));
-  }
-
-  @Test
-  public void testCompactionWithDelFiles() throws Exception {
-    resetConf();
-    int count = 4;
-    // generate mob files
-    loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count*rowNumPerFile;
-
-    assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion,
-        countMobCells(hTable));
-    assertEquals("Before deleting: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before deleting: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-
-    createDelFile();
-
-    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("Before compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("Before compaction: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before compaction: family2 file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("Before compaction: family1 del file count", regionNum,
-        countFiles(tableName, false, family1));
-    assertEquals("Before compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-
-    // do the mob file compaction
-    MobFileCompactor compactor = new PartitionedMobFileCompactor(conf, fs, tableName, hcd1, pool);
-    compactor.compact();
-
-    assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("After compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("After compaction: family1 mob file count", regionNum,
-        countFiles(tableName, true, family1));
-    assertEquals("After compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("After compaction: family1 del file count", 0,
-      countFiles(tableName, false, family1));
-    assertEquals("After compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-    assertRefFileNameEqual(family1);
-  }
-
-  @Test
-  public void testCompactionWithDelFilesAndNotMergeAllFiles() throws Exception {
-    resetConf();
-    int mergeSize = 5000;
-    // change the mob compaction merge size
-    conf.setLong(MobConstants.MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
-
-    int count = 4;
-    // generate mob files
-    loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count*rowNumPerFile;
-
-    assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion,
-        countMobCells(hTable));
-    assertEquals("Before deleting: mob file count", regionNum * count,
-      countFiles(tableName, true, family1));
-
-    int largeFilesCount = countLargeFiles(mergeSize, family1);
-    createDelFile();
-
-    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("Before compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("Before compaction: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("Before compaction: family1 del file count", regionNum,
-        countFiles(tableName, false, family1));
-    assertEquals("Before compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-
-    // do the mob file compaction
-    MobFileCompactor compactor = new PartitionedMobFileCompactor(conf, fs, tableName, hcd1, pool);
-    compactor.compact();
-
-    assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("After compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    // After the compaction, the files smaller than the mob compaction merge size
-    // is merge to one file
-    assertEquals("After compaction: family1 mob file count", largeFilesCount + regionNum,
-        countFiles(tableName, true, family1));
-    assertEquals("After compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("After compaction: family1 del file count", regionNum,
-        countFiles(tableName, false, family1));
-    assertEquals("After compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-  }
-
-  @Test
-  public void testCompactionWithDelFilesAndWithSmallCompactionBatchSize() throws Exception {
-    resetConf();
-    int batchSize = 2;
-    conf.setInt(MobConstants.MOB_FILE_COMPACTION_BATCH_SIZE, batchSize);
-    int count = 4;
-    // generate mob files
-    loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count*rowNumPerFile;
-
-    assertEquals("Before deleting: mob row count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("Before deleting: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before deleting: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-
-    createDelFile();
-
-    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("Before compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("Before compaction: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("Before compaction: family1 del file count", regionNum,
-        countFiles(tableName, false, family1));
-    assertEquals("Before compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-
-    // do the mob file compaction
-    MobFileCompactor compactor = new PartitionedMobFileCompactor(conf, fs, tableName, hcd1, pool);
-    compactor.compact();
-
-    assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("After compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("After compaction: family1 mob file count", regionNum*(count/batchSize),
-        countFiles(tableName, true, family1));
-    assertEquals("After compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("After compaction: family1 del file count", 0,
-      countFiles(tableName, false, family1));
-    assertEquals("After compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-  }
-
-  @Test
-  public void testCompactionWithHFileLink() throws IOException, InterruptedException {
-    resetConf();
-    int count = 4;
-    // generate mob files
-    loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count*rowNumPerFile;
-
-    long tid = System.currentTimeMillis();
-    byte[] snapshotName1 = Bytes.toBytes("snaptb-" + tid);
-    // take a snapshot
-    admin.snapshot(snapshotName1, tableName);
-
-    createDelFile();
-
-    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("Before compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("Before compaction: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("Before compaction: family1 del file count", regionNum,
-        countFiles(tableName, false, family1));
-    assertEquals("Before compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-
-    // do the mob file compaction
-    MobFileCompactor compactor = new PartitionedMobFileCompactor(conf, fs, tableName, hcd1, pool);
-    compactor.compact();
-
-    assertEquals("After first compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("After first compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("After first compaction: family1 mob file count", regionNum,
-        countFiles(tableName, true, family1));
-    assertEquals("After first compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("After first compaction: family1 del file count", 0,
-      countFiles(tableName, false, family1));
-    assertEquals("After first compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-    assertEquals("After first compaction: family1 hfilelink count", 0, countHFileLinks(family1));
-    assertEquals("After first compaction: family2 hfilelink count", 0, countHFileLinks(family2));
-
-    admin.disableTable(tableName);
-    // Restore from snapshot, the hfilelink will exist in mob dir
-    admin.restoreSnapshot(snapshotName1);
-    admin.enableTable(tableName);
-
-    assertEquals("After restoring snapshot: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("After restoring snapshot: mob cells count",
-        regionNum*cellNumPerRow*rowNumPerRegion, countMobCells(hTable));
-    assertEquals("After restoring snapshot: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("After restoring snapshot: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("After restoring snapshot: family1 del file count", 0,
-        countFiles(tableName, false, family1));
-    assertEquals("After restoring snapshot: family2 del file count", 0,
-        countFiles(tableName, false, family2));
-    assertEquals("After restoring snapshot: family1 hfilelink count", regionNum*count,
-        countHFileLinks(family1));
-    assertEquals("After restoring snapshot: family2 hfilelink count", 0,
-        countHFileLinks(family2));
-
-    compactor.compact();
-
-    assertEquals("After second compaction: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("After second compaction: mob cells count",
-        regionNum*cellNumPerRow*rowNumPerRegion, countMobCells(hTable));
-    assertEquals("After second compaction: family1 mob file count", regionNum,
-        countFiles(tableName, true, family1));
-    assertEquals("After second compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("After second compaction: family1 del file count", 0,
-      countFiles(tableName, false, family1));
-    assertEquals("After second compaction: family2 del file count", 0,
-      countFiles(tableName, false, family2));
-    assertEquals("After second compaction: family1 hfilelink count", 0, countHFileLinks(family1));
-    assertEquals("After second compaction: family2 hfilelink count", 0, countHFileLinks(family2));
-    assertRefFileNameEqual(family1);
-  }
-
-  @Test
-  public void testCompactionFromAdmin() throws Exception {
-    int count = 4;
-    // generate mob files
-    loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count*rowNumPerFile;
-
-    assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion,
-        countMobCells(hTable));
-    assertEquals("Before deleting: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before deleting: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-
-    createDelFile();
-
-    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("Before compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("Before compaction: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before compaction: family2 file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("Before compaction: family1 del file count", regionNum,
-        countFiles(tableName, false, family1));
-    assertEquals("Before compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-
-    int largeFilesCount = countLargeFiles(5000, family1);
-    // do the mob file compaction
-    admin.compactMob(tableName, hcd1.getName());
-
-    waitUntilCompactionFinished(tableName);
-    assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
-      countMobRows(hTable));
-    assertEquals("After compaction: mob cells count", regionNum
-      * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(hTable));
-    assertEquals("After compaction: family1 mob file count", regionNum + largeFilesCount,
-      countFiles(tableName, true, family1));
-    assertEquals("After compaction: family2 mob file count", regionNum * count,
-      countFiles(tableName, true, family2));
-    assertEquals("After compaction: family1 del file count", regionNum,
-      countFiles(tableName, false, family1));
-    assertEquals("After compaction: family2 del file count", regionNum,
-      countFiles(tableName, false, family2));
-    assertRefFileNameEqual(family1);
-  }
-
-  @Test
-  public void testMajorCompactionFromAdmin() throws Exception {
-    int count = 4;
-    // generate mob files
-    loadData(admin, bufMut, tableName, count, rowNumPerFile);
-    int rowNumPerRegion = count*rowNumPerFile;
-
-    assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion,
-        countMobRows(hTable));
-    assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion,
-        countMobCells(hTable));
-    assertEquals("Before deleting: mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-
-    createDelFile();
-
-    assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("Before compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("Before compaction: family1 mob file count", regionNum*count,
-        countFiles(tableName, true, family1));
-    assertEquals("Before compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("Before compaction: family1 del file count", regionNum,
-        countFiles(tableName, false, family1));
-    assertEquals("Before compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-
-    // do the major mob file compaction, it will force all files to compaction
-    admin.majorCompactMob(tableName, hcd1.getName());
-
-    waitUntilCompactionFinished(tableName);
-    assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
-        countMobRows(hTable));
-    assertEquals("After compaction: mob cells count",
-        regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
-    assertEquals("After compaction: family1 mob file count", regionNum,
-        countFiles(tableName, true, family1));
-    assertEquals("After compaction: family2 mob file count", regionNum*count,
-        countFiles(tableName, true, family2));
-    assertEquals("After compaction: family1 del file count", 0,
-        countFiles(tableName, false, family1));
-    assertEquals("After compaction: family2 del file count", regionNum,
-        countFiles(tableName, false, family2));
-  }
-
-  private void waitUntilCompactionFinished(TableName tableName) throws IOException,
-    InterruptedException {
-    long finished = EnvironmentEdgeManager.currentTime() + 60000;
-    CompactionState state = admin.getMobCompactionState(tableName);
-    while (EnvironmentEdgeManager.currentTime() < finished) {
-      if (state == CompactionState.NONE) {
-        break;
-      }
-      state = admin.getMobCompactionState(tableName);
-      Thread.sleep(10);
-    }
-    assertEquals(CompactionState.NONE, state);
-  }
-
-  /**
-   * Gets the number of rows in the given table.
-   * @param table to get the  scanner
-   * @return the number of rows
-   */
-  private int countMobRows(final Table table) throws IOException {
-    Scan scan = new Scan();
-    // Do not retrieve the mob data when scanning
-    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
-    ResultScanner results = table.getScanner(scan);
-    int count = 0;
-    for (Result res : results) {
-      count++;
-    }
-    results.close();
-    return count;
-  }
-
-  /**
-   * Gets the number of cells in the given table.
-   * @param table to get the  scanner
-   * @return the number of cells
-   */
-  private int countMobCells(final Table table) throws IOException {
-    Scan scan = new Scan();
-    // Do not retrieve the mob data when scanning
-    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
-    ResultScanner results = table.getScanner(scan);
-    int count = 0;
-    for (Result res : results) {
-      for (Cell cell : res.listCells()) {
-        count++;
-      }
-    }
-    results.close();
-    return count;
-  }
-
-  /**
-   * Gets the number of files in the mob path.
-   * @param isMobFile gets number of the mob files or del files
-   * @param familyName the family name
-   * @return the number of the files
-   */
-  private int countFiles(TableName tableName, boolean isMobFile, String familyName)
-    throws IOException {
-    Path mobDirPath = MobUtils.getMobFamilyPath(
-        MobUtils.getMobRegionPath(conf, tableName), familyName);
-    int count = 0;
-    if (fs.exists(mobDirPath)) {
-      FileStatus[] files = fs.listStatus(mobDirPath);
-      for (FileStatus file : files) {
-        if (isMobFile == true) {
-          if (!StoreFileInfo.isDelFile(file.getPath())) {
-            count++;
-          }
-        } else {
-          if (StoreFileInfo.isDelFile(file.getPath())) {
-            count++;
-          }
-        }
-      }
-    }
-    return count;
-  }
-
-  private boolean verifyEncryption(TableName tableName, String familyName) throws IOException {
-    Path mobDirPath = MobUtils.getMobFamilyPath(MobUtils.getMobRegionPath(conf, tableName),
-      familyName);
-    boolean hasFiles = false;
-    if (fs.exists(mobDirPath)) {
-      FileStatus[] files = fs.listStatus(mobDirPath);
-      hasFiles = files != null && files.length > 0;
-      Assert.assertTrue(hasFiles);
-      Path path = files[0].getPath();
-      CacheConfig cacheConf = new CacheConfig(conf);
-      StoreFile sf = new StoreFile(TEST_UTIL.getTestFileSystem(), path, conf, cacheConf,
-        BloomType.NONE);
-      HFile.Reader reader = sf.createReader().getHFileReader();
-      byte[] encryptionKey = reader.getTrailer().getEncryptionKey();
-      Assert.assertTrue(null != encryptionKey);
-      Assert.assertTrue(reader.getFileContext().getEncryptionContext().getCipher().getName()
-        .equals(HConstants.CIPHER_AES));
-    }
-    return hasFiles;
-  }
-
-  /**
-   * Gets the number of HFileLink in the mob path.
-   * @param familyName the family name
-   * @return the number of the HFileLink
-   */
-  private int countHFileLinks(String familyName) throws IOException {
-    Path mobDirPath = MobUtils.getMobFamilyPath(
-        MobUtils.getMobRegionPath(conf, tableName), familyName);
-    int count = 0;
-    if (fs.exists(mobDirPath)) {
-      FileStatus[] files = fs.listStatus(mobDirPath);
-      for (FileStatus file : files) {
-        if (HFileLink.isHFileLink(file.getPath())) {
-          count++;
-        }
-      }
-    }
-    return count;
-  }
-
-  /**
-   * Gets the number of files.
-   * @param size the size of the file
-   * @param familyName the family name
-   * @return the number of files large than the size
-   */
-  private int countLargeFiles(int size, String familyName) throws IOException {
-    Path mobDirPath = MobUtils.getMobFamilyPath(
-        MobUtils.getMobRegionPath(conf, tableName), familyName);
-    int count = 0;
-    if (fs.exists(mobDirPath)) {
-      FileStatus[] files = fs.listStatus(mobDirPath);
-      for (FileStatus file : files) {
-        // ignore the del files in the mob path
-        if ((!StoreFileInfo.isDelFile(file.getPath()))
-            && (file.getLen() > size)) {
-          count++;
-        }
-      }
-    }
-    return count;
-  }
-
-  /**
-   * loads some data to the table.
-   */
-  private void loadData(Admin admin, BufferedMutator table, TableName tableName, int fileNum,
-    int rowNumPerFile) throws IOException, InterruptedException {
-    if (fileNum <= 0) {
-      throw new IllegalArgumentException();
-    }
-    for (byte k0 : KEYS) {
-      byte[] k = new byte[] { k0 };
-      for (int i = 0; i < fileNum * rowNumPerFile; i++) {
-        byte[] key = Bytes.add(k, Bytes.toBytes(i));
-        byte[] mobVal = makeDummyData(10 * (i + 1));
-        Put put = new Put(key);
-        put.setDurability(Durability.SKIP_WAL);
-        put.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), mobVal);
-        put.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf2), mobVal);
-        put.addColumn(Bytes.toBytes(family2), Bytes.toBytes(qf1), mobVal);
-        table.mutate(put);
-        if ((i + 1) % rowNumPerFile == 0) {
-          table.flush();
-          admin.flush(tableName);
-        }
-      }
-    }
-  }
-
-  /**
-   * delete the row, family and cell to create the del file
-   */
-  private void createDelFile() throws IOException, InterruptedException {
-    for (byte k0 : KEYS) {
-      byte[] k = new byte[] { k0 };
-      // delete a family
-      byte[] key1 = Bytes.add(k, Bytes.toBytes(0));
-      Delete delete1 = new Delete(key1);
-      delete1.addFamily(Bytes.toBytes(family1));
-      hTable.delete(delete1);
-      // delete one row
-      byte[] key2 = Bytes.add(k, Bytes.toBytes(2));
-      Delete delete2 = new Delete(key2);
-      hTable.delete(delete2);
-      // delete one cell
-      byte[] key3 = Bytes.add(k, Bytes.toBytes(4));
-      Delete delete3 = new Delete(key3);
-      delete3.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1));
-      hTable.delete(delete3);
-      admin.flush(tableName);
-      List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(
-          Bytes.toBytes(tableNameAsString));
-      for (HRegion region : regions) {
-        region.waitForFlushesAndCompactions();
-        region.compact(true);
-      }
-    }
-  }
-  /**
-   * Creates the dummy data with a specific size.
-   * @param size the size of value
-   * @return the dummy data
-   */
-  private byte[] makeDummyData(int size) {
-    byte[] dummyData = new byte[size];
-    new Random().nextBytes(dummyData);
-    return dummyData;
-  }
-
-  /**
-   * Gets the split keys
-   */
-  private byte[][] getSplitKeys() {
-    byte[][] splitKeys = new byte[KEYS.length - 1][];
-    for (int i = 0; i < splitKeys.length; ++i) {
-      splitKeys[i] = new byte[] { KEYS[i + 1] };
-    }
-    return splitKeys;
-  }
-
-  private static ExecutorService createThreadPool(Configuration conf) {
-    int maxThreads = 10;
-    long keepAliveTime = 60;
-    final SynchronousQueue<Runnable> queue = new SynchronousQueue<Runnable>();
-    ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads,
-        keepAliveTime, TimeUnit.SECONDS, queue,
-        Threads.newDaemonThreadFactory("MobFileCompactionChore"),
-        new RejectedExecutionHandler() {
-          @Override
-          public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
-            try {
-              // waiting for a thread to pick up instead of throwing exceptions.
-              queue.put(r);
-            } catch (InterruptedException e) {
-              throw new RejectedExecutionException(e);
-            }
-          }
-        });
-    ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true);
-    return pool;
-  }
-
-  private void assertRefFileNameEqual(String familyName) throws IOException {
-    Scan scan = new Scan();
-    scan.addFamily(Bytes.toBytes(familyName));
-    // Do not retrieve the mob data when scanning
-    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
-    ResultScanner results = hTable.getScanner(scan);
-    Path mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(),
-        tableName), familyName);
-    List<Path> actualFilePaths = new ArrayList<>();
-    List<Path> expectFilePaths = new ArrayList<>();
-    for (Result res : results) {
-      for (Cell cell : res.listCells()) {
-        byte[] referenceValue = CellUtil.cloneValue(cell);
-        String fileName = Bytes.toString(referenceValue, Bytes.SIZEOF_INT,
-            referenceValue.length - Bytes.SIZEOF_INT);
-        Path targetPath = new Path(mobFamilyPath, fileName);
-        if(!actualFilePaths.contains(targetPath)) {
-          actualFilePaths.add(targetPath);
-        }
-      }
-    }
-    results.close();
-    if (fs.exists(mobFamilyPath)) {
-      FileStatus[] files = fs.listStatus(mobFamilyPath);
-      for (FileStatus file : files) {
-        if (!StoreFileInfo.isDelFile(file.getPath())) {
-          expectFilePaths.add(file.getPath());
-        }
-      }
-    }
-    Collections.sort(actualFilePaths);
-    Collections.sort(expectFilePaths);
-    assertEquals(expectFilePaths, actualFilePaths);
-  }
-
-  /**
-   * Resets the configuration.
-   */
-  private void resetConf() {
-    conf.setLong(MobConstants.MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD,
-      MobConstants.DEFAULT_MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD);
-    conf.setInt(MobConstants.MOB_FILE_COMPACTION_BATCH_SIZE,
-      MobConstants.DEFAULT_MOB_FILE_COMPACTION_BATCH_SIZE);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestPartitionedMobFileCompactionRequest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestPartitionedMobFileCompactionRequest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestPartitionedMobFileCompactionRequest.java
deleted file mode 100644
index f9159aa..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestPartitionedMobFileCompactionRequest.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.mob.filecompactions;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
-import org.apache.hadoop.hbase.mob.filecompactions.PartitionedMobFileCompactionRequest.CompactionPartition;
-import org.apache.hadoop.hbase.mob.filecompactions.PartitionedMobFileCompactionRequest.CompactionPartitionId;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(SmallTests.class)
-public class TestPartitionedMobFileCompactionRequest {
-
-  @Test
-  public void testCompactedPartitionId() {
-    String startKey1 = "startKey1";
-    String startKey2 = "startKey2";
-    String date1 = "date1";
-    String date2 = "date2";
-    CompactionPartitionId partitionId1 = new CompactionPartitionId(startKey1, date1);
-    CompactionPartitionId partitionId2 = new CompactionPartitionId(startKey2, date2);
-    CompactionPartitionId partitionId3 = new CompactionPartitionId(startKey1, date2);
-
-    Assert.assertTrue(partitionId1.equals(partitionId1));
-    Assert.assertFalse(partitionId1.equals(partitionId2));
-    Assert.assertFalse(partitionId1.equals(partitionId3));
-    Assert.assertFalse(partitionId2.equals(partitionId3));
-
-    Assert.assertEquals(startKey1, partitionId1.getStartKey());
-    Assert.assertEquals(date1, partitionId1.getDate());
-  }
-
-  @Test
-  public void testCompactedPartition() {
-    CompactionPartitionId partitionId = new CompactionPartitionId("startKey1", "date1");
-    CompactionPartition partition = new CompactionPartition(partitionId);
-    FileStatus file = new FileStatus(1, false, 1, 1024, 1, new Path("/test"));
-    partition.addFile(file);
-    Assert.assertEquals(file, partition.listFiles().get(0));
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestPartitionedMobFileCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestPartitionedMobFileCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestPartitionedMobFileCompactor.java
deleted file mode 100644
index 544d145..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/filecompactions/TestPartitionedMobFileCompactor.java
+++ /dev/null
@@ -1,436 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.mob.filecompactions;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.List;
-import java.util.Random;
-import java.util.UUID;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.RejectedExecutionHandler;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.KeyValue.Type;
-import org.apache.hadoop.hbase.regionserver.*;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.io.hfile.CacheConfig;
-import org.apache.hadoop.hbase.io.hfile.HFileContext;
-import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobFileName;
-import org.apache.hadoop.hbase.mob.MobUtils;
-import org.apache.hadoop.hbase.mob.filecompactions.MobFileCompactionRequest.CompactionType;
-import org.apache.hadoop.hbase.mob.filecompactions.PartitionedMobFileCompactionRequest.CompactionPartition;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.Threads;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-@Category(LargeTests.class)
-public class TestPartitionedMobFileCompactor {
-  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-  private final static String family = "family";
-  private final static String qf = "qf";
-  private HColumnDescriptor hcd = new HColumnDescriptor(family);
-  private Configuration conf = TEST_UTIL.getConfiguration();
-  private CacheConfig cacheConf = new CacheConfig(conf);
-  private FileSystem fs;
-  private List<FileStatus> mobFiles = new ArrayList<>();
-  private List<FileStatus> delFiles = new ArrayList<>();
-  private List<FileStatus> allFiles = new ArrayList<>();
-  private Path basePath;
-  private String mobSuffix;
-  private String delSuffix;
-  private static ExecutorService pool;
-
-  @BeforeClass
-  public static void setUpBeforeClass() throws Exception {
-    TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
-    TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
-    TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3);
-    TEST_UTIL.startMiniCluster(1);
-    pool = createThreadPool();
-  }
-
-  @AfterClass
-  public static void tearDownAfterClass() throws Exception {
-    pool.shutdown();
-    TEST_UTIL.shutdownMiniCluster();
-  }
-
-  private void init(String tableName) throws Exception {
-    fs = FileSystem.get(conf);
-    Path testDir = FSUtils.getRootDir(conf);
-    Path mobTestDir = new Path(testDir, MobConstants.MOB_DIR_NAME);
-    basePath = new Path(new Path(mobTestDir, tableName), family);
-    mobSuffix = UUID.randomUUID().toString().replaceAll("-", "");
-    delSuffix = UUID.randomUUID().toString().replaceAll("-", "") + "_del";
-  }
-
-  @Test
-  public void testCompactionSelectWithAllFiles() throws Exception {
-    resetConf();
-    String tableName = "testCompactionSelectWithAllFiles";
-    init(tableName);
-    int count = 10;
-    // create 10 mob files.
-    createStoreFiles(basePath, family, qf, count, Type.Put);
-    // create 10 del files
-    createStoreFiles(basePath, family, qf, count, Type.Delete);
-    listFiles();
-    long mergeSize = MobConstants.DEFAULT_MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD;
-    List<String> expectedStartKeys = new ArrayList<>();
-    for(FileStatus file : mobFiles) {
-      if(file.getLen() < mergeSize) {
-        String fileName = file.getPath().getName();
-        String startKey = fileName.substring(0, 32);
-        expectedStartKeys.add(startKey);
-      }
-    }
-    testSelectFiles(tableName, CompactionType.ALL_FILES, false, expectedStartKeys);
-  }
-
-  @Test
-  public void testCompactionSelectWithPartFiles() throws Exception {
-    resetConf();
-    String tableName = "testCompactionSelectWithPartFiles";
-    init(tableName);
-    int count = 10;
-    // create 10 mob files.
-    createStoreFiles(basePath, family, qf, count, Type.Put);
-    // create 10 del files
-    createStoreFiles(basePath, family, qf, count, Type.Delete);
-    listFiles();
-    long mergeSize = 4000;
-    List<String> expectedStartKeys = new ArrayList<>();
-    for(FileStatus file : mobFiles) {
-      if(file.getLen() < 4000) {
-        String fileName = file.getPath().getName();
-        String startKey = fileName.substring(0, 32);
-        expectedStartKeys.add(startKey);
-      }
-    }
-    // set the mob file compaction mergeable threshold
-    conf.setLong(MobConstants.MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
-    testSelectFiles(tableName, CompactionType.PART_FILES, false, expectedStartKeys);
-  }
-
-  @Test
-  public void testCompactionSelectWithForceAllFiles() throws Exception {
-    resetConf();
-    String tableName = "testCompactionSelectWithForceAllFiles";
-    init(tableName);
-    int count = 10;
-    // create 10 mob files.
-    createStoreFiles(basePath, family, qf, count, Type.Put);
-    // create 10 del files
-    createStoreFiles(basePath, family, qf, count, Type.Delete);
-    listFiles();
-    long mergeSize = 4000;
-    List<String> expectedStartKeys = new ArrayList<>();
-    for(FileStatus file : mobFiles) {
-      String fileName = file.getPath().getName();
-      String startKey = fileName.substring(0, 32);
-      expectedStartKeys.add(startKey);
-    }
-    // set the mob file compaction mergeable threshold
-    conf.setLong(MobConstants.MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
-    testSelectFiles(tableName, CompactionType.ALL_FILES, true, expectedStartKeys);
-  }
-
-  @Test
-  public void testCompactDelFilesWithDefaultBatchSize() throws Exception {
-    resetConf();
-    String tableName = "testCompactDelFilesWithDefaultBatchSize";
-    init(tableName);
-    // create 20 mob files.
-    createStoreFiles(basePath, family, qf, 20, Type.Put);
-    // create 13 del files
-    createStoreFiles(basePath, family, qf, 13, Type.Delete);
-    listFiles();
-    testCompactDelFiles(tableName, 1, 13, false);
-  }
-
-  @Test
-  public void testCompactDelFilesWithSmallBatchSize() throws Exception {
-    resetConf();
-    String tableName = "testCompactDelFilesWithSmallBatchSize";
-    init(tableName);
-    // create 20 mob files.
-    createStoreFiles(basePath, family, qf, 20, Type.Put);
-    // create 13 del files
-    createStoreFiles(basePath, family, qf, 13, Type.Delete);
-    listFiles();
-
-    // set the mob file compaction batch size
-    conf.setInt(MobConstants.MOB_FILE_COMPACTION_BATCH_SIZE, 4);
-    testCompactDelFiles(tableName, 1, 13, false);
-  }
-
-  @Test
-  public void testCompactDelFilesChangeMaxDelFileCount() throws Exception {
-    resetConf();
-    String tableName = "testCompactDelFilesWithSmallBatchSize";
-    init(tableName);
-    // create 20 mob files.
-    createStoreFiles(basePath, family, qf, 20, Type.Put);
-    // create 13 del files
-    createStoreFiles(basePath, family, qf, 13, Type.Delete);
-    listFiles();
-
-    // set the max del file count
-    conf.setInt(MobConstants.MOB_DELFILE_MAX_COUNT, 5);
-    // set the mob file compaction batch size
-    conf.setInt(MobConstants.MOB_FILE_COMPACTION_BATCH_SIZE, 2);
-    testCompactDelFiles(tableName, 4, 13, false);
-  }
-
-  /**
-   * Tests the selectFiles
-   * @param tableName the table name
-   * @param type the expected compaction type
-   * @param expected the expected start keys
-   */
-  private void testSelectFiles(String tableName, final CompactionType type,
-    final boolean isForceAllFiles, final List<String> expected) throws IOException {
-    PartitionedMobFileCompactor compactor = new PartitionedMobFileCompactor(conf, fs,
-      TableName.valueOf(tableName), hcd, pool) {
-      @Override
-      public List<Path> compact(List<FileStatus> files, boolean isForceAllFiles)
-        throws IOException {
-        if (files == null || files.isEmpty()) {
-          return null;
-        }
-        PartitionedMobFileCompactionRequest request = select(files, isForceAllFiles);
-        // assert the compaction type
-        Assert.assertEquals(type, request.type);
-        // assert get the right partitions
-        compareCompactedPartitions(expected, request.compactionPartitions);
-        // assert get the right del files
-        compareDelFiles(request.delFiles);
-        return null;
-      }
-    };
-    compactor.compact(allFiles, isForceAllFiles);
-  }
-
-  /**
-   * Tests the compacteDelFile
-   * @param tableName the table name
-   * @param expectedFileCount the expected file count
-   * @param expectedCellCount the expected cell count
-   */
-  private void testCompactDelFiles(String tableName, final int expectedFileCount,
-      final int expectedCellCount, boolean isForceAllFiles) throws IOException {
-    PartitionedMobFileCompactor compactor = new PartitionedMobFileCompactor(conf, fs,
-      TableName.valueOf(tableName), hcd, pool) {
-      @Override
-      protected List<Path> performCompaction(PartitionedMobFileCompactionRequest request)
-          throws IOException {
-        List<Path> delFilePaths = new ArrayList<Path>();
-        for (FileStatus delFile : request.delFiles) {
-          delFilePaths.add(delFile.getPath());
-        }
-        List<Path> newDelPaths = compactDelFiles(request, delFilePaths);
-        // assert the del files are merged.
-        Assert.assertEquals(expectedFileCount, newDelPaths.size());
-        Assert.assertEquals(expectedCellCount, countDelCellsInDelFiles(newDelPaths));
-        return null;
-      }
-    };
-    compactor.compact(allFiles, isForceAllFiles);
-  }
-
-  /**
-   * Lists the files in the path
-   */
-  private void listFiles() throws IOException {
-    for (FileStatus file : fs.listStatus(basePath)) {
-      allFiles.add(file);
-      if (file.getPath().getName().endsWith("_del")) {
-        delFiles.add(file);
-      } else {
-        mobFiles.add(file);
-      }
-    }
-  }
-
-  /**
-   * Compares the compacted partitions.
-   * @param partitions the collection of CompactedPartitions
-   */
-  private void compareCompactedPartitions(List<String> expected,
-      Collection<CompactionPartition> partitions) {
-    List<String> actualKeys = new ArrayList<>();
-    for (CompactionPartition partition : partitions) {
-      actualKeys.add(partition.getPartitionId().getStartKey());
-    }
-    Collections.sort(expected);
-    Collections.sort(actualKeys);
-    Assert.assertEquals(expected.size(), actualKeys.size());
-    for (int i = 0; i < expected.size(); i++) {
-      Assert.assertEquals(expected.get(i), actualKeys.get(i));
-    }
-  }
-
-  /**
-   * Compares the del files.
-   * @param allDelFiles all the del files
-   */
-  private void compareDelFiles(Collection<FileStatus> allDelFiles) {
-    int i = 0;
-    for (FileStatus file : allDelFiles) {
-      Assert.assertEquals(delFiles.get(i), file);
-      i++;
-    }
-  }
-
-  /**
-   * Creates store files.
-   * @param basePath the path to create file
-   * @family the family name
-   * @qualifier the column qualifier
-   * @count the store file number
-   * @type the key type
-   */
-  private void createStoreFiles(Path basePath, String family, String qualifier, int count,
-      Type type) throws IOException {
-    HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
-    String startKey = "row_";
-    MobFileName mobFileName = null;
-    for (int i = 0; i < count; i++) {
-      byte[] startRow = Bytes.toBytes(startKey + i) ;
-      if(type.equals(Type.Delete)) {
-        mobFileName = MobFileName.create(startRow, MobUtils.formatDate(
-            new Date()), delSuffix);
-      }
-      if(type.equals(Type.Put)){
-        mobFileName = MobFileName.create(Bytes.toBytes(startKey + i), MobUtils.formatDate(
-            new Date()), mobSuffix);
-      }
-      StoreFile.Writer mobFileWriter = new StoreFile.WriterBuilder(conf, cacheConf, fs)
-      .withFileContext(meta).withFilePath(new Path(basePath, mobFileName.getFileName())).build();
-      writeStoreFile(mobFileWriter, startRow, Bytes.toBytes(family), Bytes.toBytes(qualifier),
-          type, (i+1)*1000);
-    }
-  }
-
-  /**
-   * Writes data to store file.
-   * @param writer the store file writer
-   * @param row the row key
-   * @param family the family name
-   * @param qualifier the column qualifier
-   * @param type the key type
-   * @param size the size of value
-   */
-  private static void writeStoreFile(final StoreFile.Writer writer, byte[]row, byte[] family,
-      byte[] qualifier, Type type, int size) throws IOException {
-    long now = System.currentTimeMillis();
-    try {
-      byte[] dummyData = new byte[size];
-      new Random().nextBytes(dummyData);
-      writer.append(new KeyValue(row, family, qualifier, now, type, dummyData));
-    } finally {
-      writer.close();
-    }
-  }
-
-  /**
-   * Gets the number of del cell in the del files
-   * @param paths the del file paths
-   * @return the cell size
-   */
-  private int countDelCellsInDelFiles(List<Path> paths) throws IOException {
-    List<StoreFile> sfs = new ArrayList<StoreFile>();
-    int size = 0;
-    for(Path path : paths) {
-      StoreFile sf = new StoreFile(fs, path, conf, cacheConf, BloomType.NONE);
-      sfs.add(sf);
-    }
-    List scanners = StoreFileScanner.getScannersForStoreFiles(sfs, false, true,
-        false, null, HConstants.LATEST_TIMESTAMP);
-    Scan scan = new Scan();
-    scan.setMaxVersions(hcd.getMaxVersions());
-    long timeToPurgeDeletes = Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);
-    long ttl = HStore.determineTTLFromFamily(hcd);
-    ScanInfo scanInfo = new ScanInfo(hcd, ttl, timeToPurgeDeletes, CellComparator.COMPARATOR);
-    StoreScanner scanner = new StoreScanner(scan, scanInfo, ScanType.COMPACT_RETAIN_DELETES, null,
-        scanners, 0L, HConstants.LATEST_TIMESTAMP);
-    List<Cell> results = new ArrayList<>();
-    boolean hasMore = true;
-
-    while (hasMore) {
-      hasMore = scanner.next(results);
-      size += results.size();
-      results.clear();
-    }
-    scanner.close();
-    return size;
-  }
-
-  private static ExecutorService createThreadPool() {
-    int maxThreads = 10;
-    long keepAliveTime = 60;
-    final SynchronousQueue<Runnable> queue = new SynchronousQueue<Runnable>();
-    ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads, keepAliveTime,
-      TimeUnit.SECONDS, queue, Threads.newDaemonThreadFactory("MobFileCompactionChore"),
-      new RejectedExecutionHandler() {
-        @Override
-        public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
-          try {
-            // waiting for a thread to pick up instead of throwing exceptions.
-            queue.put(r);
-          } catch (InterruptedException e) {
-            throw new RejectedExecutionException(e);
-          }
-        }
-      });
-    ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true);
-    return pool;
-  }
-
-  /**
-   * Resets the configuration.
-   */
-  private void resetConf() {
-    conf.setLong(MobConstants.MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD,
-      MobConstants.DEFAULT_MOB_FILE_COMPACTION_MERGEABLE_THRESHOLD);
-    conf.setInt(MobConstants.MOB_DELFILE_MAX_COUNT, MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT);
-    conf.setInt(MobConstants.MOB_FILE_COMPACTION_BATCH_SIZE,
-      MobConstants.DEFAULT_MOB_FILE_COMPACTION_BATCH_SIZE);
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
index 67fcf96..bc5f494 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
@@ -267,22 +267,22 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
   }
 
   @Override
-  public long getMobCompactedIntoMobCellsCount() {
+  public long getCellsCountCompactedToMob() {
     return 20;
   }
 
   @Override
-  public long getMobCompactedFromMobCellsCount() {
+  public long getCellsCountCompactedFromMob() {
     return 10;
   }
 
   @Override
-  public long getMobCompactedIntoMobCellsSize() {
+  public long getCellsSizeCompactedToMob() {
     return 200;
   }
 
   @Override
-  public long getMobCompactedFromMobCellsSize() {
+  public long getCellsSizeCompactedFromMob() {
     return 100;
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/b31a6acf/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobCompaction.java
deleted file mode 100644
index 10b680a..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobCompaction.java
+++ /dev/null
@@ -1,460 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.regionserver;
-
-import static org.apache.hadoop.hbase.HBaseTestingUtility.START_KEY;
-import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.HBaseTestCase.HRegionIncommon;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.io.hfile.CacheConfig;
-import org.apache.hadoop.hbase.io.hfile.HFile;
-import org.apache.hadoop.hbase.io.hfile.HFileContext;
-import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
-import org.apache.hadoop.hbase.mob.MobConstants;
-import org.apache.hadoop.hbase.mob.MobUtils;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.util.Pair;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
-
-/**
- * Test mob compaction
- */
-@Category(MediumTests.class)
-public class TestMobCompaction {
-  @Rule
-  public TestName name = new TestName();
-  static final Log LOG = LogFactory.getLog(TestMobCompaction.class.getName());
-  private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
-  private Configuration conf = null;
-
-  private HRegion region = null;
-  private HTableDescriptor htd = null;
-  private HColumnDescriptor hcd = null;
-  private long mobCellThreshold = 1000;
-
-  private FileSystem fs;
-
-  private static final byte[] COLUMN_FAMILY = fam1;
-  private final byte[] STARTROW = Bytes.toBytes(START_KEY);
-  private int compactionThreshold;
-
-  @BeforeClass
-  public static void setUpBeforeClass() throws Exception {
-    UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
-    UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
-    UTIL.startMiniCluster(1);
-  }
-
-  @AfterClass
-  public static void tearDownAfterClass() throws Exception {
-    UTIL.shutdownMiniCluster();
-  }
-
-  private void init(Configuration conf, long mobThreshold) throws Exception {
-    this.conf = conf;
-    this.mobCellThreshold = mobThreshold;
-    HBaseTestingUtility UTIL = new HBaseTestingUtility(conf);
-
-    compactionThreshold = conf.getInt("hbase.hstore.compactionThreshold", 3);
-    htd = UTIL.createTableDescriptor(name.getMethodName());
-    hcd = new HColumnDescriptor(COLUMN_FAMILY);
-    hcd.setMobEnabled(true);
-    hcd.setMobThreshold(mobThreshold);
-    hcd.setMaxVersions(1);
-    htd.modifyFamily(hcd);
-
-    region = UTIL.createLocalHRegion(htd, null, null);
-    fs = FileSystem.get(conf);
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    region.close();
-    fs.delete(UTIL.getDataTestDir(), true);
-  }
-
-  /**
-   * During compaction, cells smaller than the threshold won't be affected.
-   */
-  @Test
-  public void testSmallerValue() throws Exception {
-    init(UTIL.getConfiguration(), 500);
-    byte[] dummyData = makeDummyData(300); // smaller than mob threshold
-    HRegionIncommon loader = new HRegionIncommon(region);
-    // one hfile per row
-    for (int i = 0; i < compactionThreshold; i++) {
-      Put p = createPut(i, dummyData);
-      loader.put(p);
-      loader.flushcache();
-    }
-    assertEquals("Before compaction: store files", compactionThreshold, countStoreFiles());
-    assertEquals("Before compaction: mob file count", 0, countMobFiles());
-    assertEquals("Before compaction: rows", compactionThreshold, countRows());
-    assertEquals("Before compaction: mob rows", 0, countMobRows());
-
-    region.compactStores();
-
-    assertEquals("After compaction: store files", 1, countStoreFiles());
-    assertEquals("After compaction: mob file count", 0, countMobFiles());
-    assertEquals("After compaction: referenced mob file count", 0, countReferencedMobFiles());
-    assertEquals("After compaction: rows", compactionThreshold, countRows());
-    assertEquals("After compaction: mob rows", 0, countMobRows());
-  }
-
-  /**
-   * During compaction, the mob threshold size is changed.
-   */
-  @Test
-  public void testLargerValue() throws Exception {
-    init(UTIL.getConfiguration(), 200);
-    byte[] dummyData = makeDummyData(300); // larger than mob threshold
-    HRegionIncommon loader = new HRegionIncommon(region);
-    for (int i = 0; i < compactionThreshold; i++) {
-      Put p = createPut(i, dummyData);
-      loader.put(p);
-      loader.flushcache();
-    }
-    assertEquals("Before compaction: store files", compactionThreshold, countStoreFiles());
-    assertEquals("Before compaction: mob file count", compactionThreshold, countMobFiles());
-    assertEquals("Before compaction: rows", compactionThreshold, countRows());
-    assertEquals("Before compaction: mob rows", compactionThreshold, countMobRows());
-    assertEquals("Before compaction: number of mob cells", compactionThreshold,
-        countMobCellsInMetadata());
-    // Change the threshold larger than the data size
-    region.getTableDesc().getFamily(COLUMN_FAMILY).setMobThreshold(500);
-    region.initialize();
-    region.compactStores();
-
-    assertEquals("After compaction: store files", 1, countStoreFiles());
-    assertEquals("After compaction: mob file count", compactionThreshold, countMobFiles());
-    assertEquals("After compaction: referenced mob file count", 0, countReferencedMobFiles());
-    assertEquals("After compaction: rows", compactionThreshold, countRows());
-    assertEquals("After compaction: mob rows", 0, countMobRows());
-  }
-
-  /**
-   * This test will first generate store files, then bulk load them and trigger the compaction. When
-   * compaction, the cell value will be larger than the threshold.
-   */
-  @Test
-  public void testMobCompactionWithBulkload() throws Exception {
-    // The following will produce store files of 600.
-    init(UTIL.getConfiguration(), 300);
-    byte[] dummyData = makeDummyData(600);
-
-    Path hbaseRootDir = FSUtils.getRootDir(conf);
-    Path basedir = new Path(hbaseRootDir, htd.getNameAsString());
-    List<Pair<byte[], String>> hfiles = new ArrayList<>(1);
-    for (int i = 0; i < compactionThreshold; i++) {
-      Path hpath = new Path(basedir, "hfile" + i);
-      hfiles.add(Pair.newPair(COLUMN_FAMILY, hpath.toString()));
-      createHFile(hpath, i, dummyData);
-    }
-
-    // The following will bulk load the above generated store files and compact, with 600(fileSize)
-    // > 300(threshold)
-    boolean result = region.bulkLoadHFiles(hfiles, true, null);
-    assertTrue("Bulkload result:", result);
-    assertEquals("Before compaction: store files", compactionThreshold, countStoreFiles());
-    assertEquals("Before compaction: mob file count", 0, countMobFiles());
-    assertEquals("Before compaction: rows", compactionThreshold, countRows());
-    assertEquals("Before compaction: mob rows", 0, countMobRows());
-    assertEquals("Before compaction: referenced mob file count", 0, countReferencedMobFiles());
-
-    region.compactStores();
-
-    assertEquals("After compaction: store files", 1, countStoreFiles());
-    assertEquals("After compaction: mob file count:", 1, countMobFiles());
-    assertEquals("After compaction: rows", compactionThreshold, countRows());
-    assertEquals("After compaction: mob rows", compactionThreshold, countMobRows());
-    assertEquals("After compaction: referenced mob file count", 1, countReferencedMobFiles());
-    assertEquals("After compaction: number of mob cells", compactionThreshold,
-        countMobCellsInMetadata());
-  }
-
-  @Test
-  public void testMajorCompactionAfterDelete() throws Exception {
-    init(UTIL.getConfiguration(), 100);
-    byte[] dummyData = makeDummyData(200); // larger than mob threshold
-    HRegionIncommon loader = new HRegionIncommon(region);
-    // create hfiles and mob hfiles but don't trigger compaction
-    int numHfiles = compactionThreshold - 1;
-    byte[] deleteRow = Bytes.add(STARTROW, Bytes.toBytes(0));
-    for (int i = 0; i < numHfiles; i++) {
-      Put p = createPut(i, dummyData);
-      loader.put(p);
-      loader.flushcache();
-    }
-    assertEquals("Before compaction: store files", numHfiles, countStoreFiles());
-    assertEquals("Before compaction: mob file count", numHfiles, countMobFiles());
-    assertEquals("Before compaction: rows", numHfiles, countRows());
-    assertEquals("Before compaction: mob rows", numHfiles, countMobRows());
-    assertEquals("Before compaction: number of mob cells", numHfiles, countMobCellsInMetadata());
-    // now let's delete some cells that contain mobs
-    Delete delete = new Delete(deleteRow);
-    delete.addFamily(COLUMN_FAMILY);
-    region.delete(delete);
-    loader.flushcache();
-
-    assertEquals("Before compaction: store files", numHfiles + 1, countStoreFiles());
-    assertEquals("Before compaction: mob files", numHfiles, countMobFiles());
-    // region.compactStores();
-    region.compact(true);
-    assertEquals("After compaction: store files", 1, countStoreFiles());
-    // still have original mob hfiles and now added a mob del file
-    assertEquals("After compaction: mob files", numHfiles + 1, countMobFiles());
-
-    Scan scan = new Scan();
-    scan.setRaw(true);
-    InternalScanner scanner = region.getScanner(scan);
-    List<Cell> results = new ArrayList<>();
-    scanner.next(results);
-    int deleteCount = 0;
-    while (!results.isEmpty()) {
-      for (Cell c : results) {
-        if (c.getTypeByte() == KeyValue.Type.DeleteFamily.getCode()) {
-          deleteCount++;
-          assertTrue(Bytes.equals(CellUtil.cloneRow(c), deleteRow));
-        }
-      }
-      results.clear();
-      scanner.next(results);
-    }
-    // assert the delete mark is not retained after the major compaction
-    assertEquals(0, deleteCount);
-    scanner.close();
-    // assert the deleted cell is not counted
-    assertEquals("The cells in mob files", numHfiles - 1, countMobCellsInMobFiles(1));
-  }
-
-  private int countStoreFiles() throws IOException {
-    Store store = region.getStore(COLUMN_FAMILY);
-    return store.getStorefilesCount();
-  }
-
-  private int countMobFiles() throws IOException {
-    Path mobDirPath = new Path(MobUtils.getMobRegionPath(conf, htd.getTableName()),
-        hcd.getNameAsString());
-    if (fs.exists(mobDirPath)) {
-      FileStatus[] files = UTIL.getTestFileSystem().listStatus(mobDirPath);
-      return files.length;
-    }
-    return 0;
-  }
-
-  private long countMobCellsInMetadata() throws IOException {
-    long mobCellsCount = 0;
-    Path mobDirPath = new Path(MobUtils.getMobRegionPath(conf, htd.getTableName()),
-        hcd.getNameAsString());
-    Configuration copyOfConf = new Configuration(conf);
-    copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f);
-    CacheConfig cacheConfig = new CacheConfig(copyOfConf);
-    if (fs.exists(mobDirPath)) {
-      FileStatus[] files = UTIL.getTestFileSystem().listStatus(mobDirPath);
-      for (FileStatus file : files) {
-        StoreFile sf = new StoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE);
-        Map<byte[], byte[]> fileInfo = sf.createReader().loadFileInfo();
-        byte[] count = fileInfo.get(StoreFile.MOB_CELLS_COUNT);
-        assertTrue(count != null);
-        mobCellsCount += Bytes.toLong(count);
-      }
-    }
-    return mobCellsCount;
-  }
-
-  private Put createPut(int rowIdx, byte[] dummyData) throws IOException {
-    Put p = new Put(Bytes.add(STARTROW, Bytes.toBytes(rowIdx)));
-    p.setDurability(Durability.SKIP_WAL);
-    p.addColumn(COLUMN_FAMILY, Bytes.toBytes("colX"), dummyData);
-    return p;
-  }
-
-  /**
-   * Create an HFile with the given number of bytes
-   */
-  private void createHFile(Path path, int rowIdx, byte[] dummyData) throws IOException {
-    HFileContext meta = new HFileContextBuilder().build();
-    HFile.Writer writer = HFile.getWriterFactory(conf, new CacheConfig(conf)).withPath(fs, path)
-        .withFileContext(meta).create();
-    long now = System.currentTimeMillis();
-    try {
-      KeyValue kv = new KeyValue(Bytes.add(STARTROW, Bytes.toBytes(rowIdx)), COLUMN_FAMILY,
-          Bytes.toBytes("colX"), now, dummyData);
-      writer.append(kv);
-    } finally {
-      writer.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis()));
-      writer.close();
-    }
-  }
-
-  private int countMobRows() throws IOException {
-    Scan scan = new Scan();
-    // Do not retrieve the mob data when scanning
-    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
-    InternalScanner scanner = region.getScanner(scan);
-
-    int scannedCount = 0;
-    List<Cell> results = new ArrayList<>();
-    boolean hasMore = true;
-    while (hasMore) {
-      hasMore = scanner.next(results);
-      for (Cell c : results) {
-        if (MobUtils.isMobReferenceCell(c)) {
-          scannedCount++;
-        }
-      }
-      results.clear();
-    }
-    scanner.close();
-
-    return scannedCount;
-  }
-
-  private int countRows() throws IOException {
-    Scan scan = new Scan();
-    // Do not retrieve the mob data when scanning
-    InternalScanner scanner = region.getScanner(scan);
-
-    int scannedCount = 0;
-    List<Cell> results = new ArrayList<Cell>();
-    boolean hasMore = true;
-    while (hasMore) {
-      hasMore = scanner.next(results);
-      scannedCount += results.size();
-      results.clear();
-    }
-    scanner.close();
-
-    return scannedCount;
-  }
-
-  private byte[] makeDummyData(int size) {
-    byte[] dummyData = new byte[size];
-    new Random().nextBytes(dummyData);
-    return dummyData;
-  }
-
-  private int countReferencedMobFiles() throws IOException {
-    Scan scan = new Scan();
-    // Do not retrieve the mob data when scanning
-    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
-    InternalScanner scanner = region.getScanner(scan);
-
-    List<Cell> kvs = new ArrayList<>();
-    boolean hasMore = true;
-    String fileName;
-    Set<String> files = new HashSet<>();
-    do {
-      kvs.clear();
-      hasMore = scanner.next(kvs);
-      for (Cell kv : kvs) {
-        if (!MobUtils.isMobReferenceCell(kv)) {
-          continue;
-        }
-        if (!MobUtils.hasValidMobRefCellValue(kv)) {
-          continue;
-        }
-        int size = MobUtils.getMobValueLength(kv);
-        if (size <= mobCellThreshold) {
-          continue;
-        }
-        fileName = MobUtils.getMobFileName(kv);
-        if (fileName.isEmpty()) {
-          continue;
-        }
-        files.add(fileName);
-        Path familyPath = MobUtils.getMobFamilyPath(conf, htd.getTableName(),
-            hcd.getNameAsString());
-        assertTrue(fs.exists(new Path(familyPath, fileName)));
-      }
-    } while (hasMore);
-
-    scanner.close();
-
-    return files.size();
-  }
-
-  private int countMobCellsInMobFiles(int expectedNumDelfiles) throws IOException {
-    Configuration copyOfConf = new Configuration(conf);
-    copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f);
-    CacheConfig cacheConfig = new CacheConfig(copyOfConf);
-    Path mobDirPath = new Path(MobUtils.getMobRegionPath(conf, htd.getTableName()),
-        hcd.getNameAsString());
-    List<StoreFile> sfs = new ArrayList<>();
-    int numDelfiles = 0;
-    int size = 0;
-    if (fs.exists(mobDirPath)) {
-      for (FileStatus f : fs.listStatus(mobDirPath)) {
-        StoreFile sf = new StoreFile(fs, f.getPath(), conf, cacheConfig, BloomType.NONE);
-        sfs.add(sf);
-        if (StoreFileInfo.isDelFile(sf.getPath())) {
-          numDelfiles++;
-        }
-      }
-      List scanners = StoreFileScanner.getScannersForStoreFiles(sfs, false, true, false, null,
-          HConstants.LATEST_TIMESTAMP);
-      Scan scan = new Scan();
-      scan.setMaxVersions(hcd.getMaxVersions());
-      long timeToPurgeDeletes = Math.max(conf.getLong("hbase.hstore.time.to.purge.deletes", 0), 0);
-      long ttl = HStore.determineTTLFromFamily(hcd);
-      ScanInfo scanInfo = new ScanInfo(hcd, ttl, timeToPurgeDeletes, CellComparator.COMPARATOR);
-      StoreScanner scanner = new StoreScanner(scan, scanInfo, ScanType.COMPACT_DROP_DELETES, null,
-          scanners, 0L, HConstants.LATEST_TIMESTAMP);
-      List<Cell> results = new ArrayList<>();
-      boolean hasMore = true;
-      while (hasMore) {
-        hasMore = scanner.next(results);
-        size += results.size();
-        results.clear();
-      }
-    }
-    // assert the number of the existing del files
-    assertEquals(expectedNumDelfiles, numDelfiles);
-    return size;
-  }
-}


Mime
View raw message