hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jmhs...@apache.org
Subject [1/2] HBASE-11643 Read and write MOB in HBase (Jingcheng Du)
Date Thu, 04 Sep 2014 12:43:54 GMT
Repository: hbase
Updated Branches:
  refs/heads/hbase-11339 [created] 5c14f749b


http://git-wip-us.apache.org/repos/asf/hbase/blob/5c14f749/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java
new file mode 100644
index 0000000..5e28cd9
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java
@@ -0,0 +1,86 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mob;
+
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Assert;
+
+public class MobTestUtil {
+  protected static final char FIRST_CHAR = 'a';
+  protected static final char LAST_CHAR = 'z';
+
+  protected static String generateRandomString(int demoLength) {
+    String base = "abcdefghijklmnopqrstuvwxyz";
+    Random random = new Random();
+    StringBuffer sb = new StringBuffer();
+    for (int i = 0; i < demoLength; i++) {
+      int number = random.nextInt(base.length());
+      sb.append(base.charAt(number));
+    }
+    return sb.toString();
+  }
+  protected static void writeStoreFile(final StoreFile.Writer writer, String caseName)
+      throws IOException {
+    writeStoreFile(writer, Bytes.toBytes(caseName), Bytes.toBytes(caseName));
+  }
+
+  /*
+   * Writes HStoreKey and ImmutableBytes data to passed writer and then closes
+   * it.
+   *
+   * @param writer
+   *
+   * @throws IOException
+   */
+  private static void writeStoreFile(final StoreFile.Writer writer, byte[] fam,
+      byte[] qualifier) throws IOException {
+    long now = System.currentTimeMillis();
+    try {
+      for (char d = FIRST_CHAR; d <= LAST_CHAR; d++) {
+        for (char e = FIRST_CHAR; e <= LAST_CHAR; e++) {
+          byte[] b = new byte[] { (byte) d, (byte) e };
+          writer.append(new KeyValue(b, fam, qualifier, now, b));
+        }
+      }
+    } finally {
+      writer.close();
+    }
+  }
+
+  /**
+   * Compare two KeyValue only for their row family qualifier value
+   */
+  public static void assertKeyValuesEquals(KeyValue firstKeyValue,
+	      KeyValue secondKeyValue) {
+		    Assert.assertEquals(Bytes.toString(CellUtil.cloneRow(firstKeyValue)),
+	            Bytes.toString(CellUtil.cloneRow(secondKeyValue)));
+		    Assert.assertEquals(Bytes.toString(CellUtil.cloneFamily(firstKeyValue)),
+	            Bytes.toString(CellUtil.cloneFamily(secondKeyValue)));
+		    Assert.assertEquals(Bytes.toString(CellUtil.cloneQualifier(firstKeyValue)),
+	            Bytes.toString(CellUtil.cloneQualifier(secondKeyValue)));
+		    Assert.assertEquals(Bytes.toString(CellUtil.cloneValue(firstKeyValue)),
+	            Bytes.toString(CellUtil.cloneValue(secondKeyValue)));
+	  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/5c14f749/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java
new file mode 100644
index 0000000..b39dd2a
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestCachedMobFile.java
@@ -0,0 +1,154 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mob;
+
+import junit.framework.TestCase;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValue.Type;
+import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.HFileContext;
+import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestCachedMobFile extends TestCase{
+  static final Log LOG = LogFactory.getLog(TestCachedMobFile.class);
+  private Configuration conf = HBaseConfiguration.create();
+  private CacheConfig cacheConf = new CacheConfig(conf);
+  private final String TABLE = "tableName";
+  private final String FAMILY = "familyName";
+  private final String FAMILY1 = "familyName1";
+  private final String FAMILY2 = "familyName2";
+  private final long EXPECTED_REFERENCE_ZERO = 0;
+  private final long EXPECTED_REFERENCE_ONE = 1;
+  private final long EXPECTED_REFERENCE_TWO = 2;
+
+  @Test
+  public void testOpenClose() throws Exception {
+    String caseName = getName();
+    FileSystem fs = FileSystem.get(conf);
+    Path testDir = FSUtils.getRootDir(conf);
+    Path outputDir = new Path(new Path(testDir, TABLE),
+        FAMILY);
+    HFileContext meta = new HFileContextBuilder().withBlockSize(8*1024).build();
+    StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs)
+        .withOutputDir(outputDir).withFileContext(meta).build();
+    MobTestUtil.writeStoreFile(writer, caseName);
+    CachedMobFile cachedMobFile = CachedMobFile.create(fs, writer.getPath(), conf, cacheConf);
+    Assert.assertEquals(EXPECTED_REFERENCE_ZERO, cachedMobFile.getReferenceCount());
+    cachedMobFile.open();
+    Assert.assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile.getReferenceCount());
+    cachedMobFile.open();
+    Assert.assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile.getReferenceCount());
+    cachedMobFile.close();
+    Assert.assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile.getReferenceCount());
+    cachedMobFile.close();
+    Assert.assertEquals(EXPECTED_REFERENCE_ZERO, cachedMobFile.getReferenceCount());
+  }
+
+  @Test
+  public void testCompare() throws Exception {
+    String caseName = getName();
+    FileSystem fs = FileSystem.get(conf);
+    Path testDir = FSUtils.getRootDir(conf);
+    Path outputDir1 = new Path(new Path(testDir, TABLE),
+        FAMILY1);
+    HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
+    StoreFile.Writer writer1 = new StoreFile.WriterBuilder(conf, cacheConf, fs)
+        .withOutputDir(outputDir1).withFileContext(meta).build();
+    MobTestUtil.writeStoreFile(writer1, caseName);
+    CachedMobFile cachedMobFile1 = CachedMobFile.create(fs, writer1.getPath(), conf, cacheConf);
+    Path outputDir2 = new Path(new Path(testDir, TABLE),
+        FAMILY2);
+    StoreFile.Writer writer2 = new StoreFile.WriterBuilder(conf, cacheConf, fs)
+    .withOutputDir(outputDir2)
+    .withFileContext(meta)
+    .build();
+    MobTestUtil.writeStoreFile(writer2, caseName);
+    CachedMobFile cachedMobFile2 = CachedMobFile.create(fs, writer2.getPath(), conf, cacheConf);
+    cachedMobFile1.access(1);
+    cachedMobFile2.access(2);
+    Assert.assertEquals(cachedMobFile1.compareTo(cachedMobFile2), 1);
+    Assert.assertEquals(cachedMobFile2.compareTo(cachedMobFile1), -1);
+    Assert.assertEquals(cachedMobFile1.compareTo(cachedMobFile1), 0);
+  }
+
+  @Test
+  public void testReadKeyValue() throws Exception {
+    FileSystem fs = FileSystem.get(conf);
+    Path testDir = FSUtils.getRootDir(conf);
+    Path outputDir = new Path(new Path(testDir, TABLE), "familyname");
+    HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build();
+    StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs)
+        .withOutputDir(outputDir).withFileContext(meta).build();
+    String caseName = getName();
+    MobTestUtil.writeStoreFile(writer, caseName);
+    CachedMobFile cachedMobFile = CachedMobFile.create(fs, writer.getPath(), conf, cacheConf);
+    byte[] family = Bytes.toBytes(caseName);
+    byte[] qualify = Bytes.toBytes(caseName);
+    // Test the start key
+    byte[] startKey = Bytes.toBytes("aa");  // The start key bytes
+    KeyValue expectedKey =
+        new KeyValue(startKey, family, qualify, Long.MAX_VALUE, Type.Put, startKey);
+    KeyValue seekKey = expectedKey.createKeyOnly(false);
+    KeyValue kv = KeyValueUtil.ensureKeyValue(cachedMobFile.readCell(seekKey, false));
+    MobTestUtil.assertKeyValuesEquals(expectedKey, kv);
+
+    // Test the end key
+    byte[] endKey = Bytes.toBytes("zz");  // The end key bytes
+    expectedKey = new KeyValue(endKey, family, qualify, Long.MAX_VALUE, Type.Put, endKey);
+    seekKey = expectedKey.createKeyOnly(false);
+    kv = KeyValueUtil.ensureKeyValue(cachedMobFile.readCell(seekKey, false));
+    MobTestUtil.assertKeyValuesEquals(expectedKey, kv);
+
+    // Test the random key
+    byte[] randomKey = Bytes.toBytes(MobTestUtil.generateRandomString(2));
+    expectedKey = new KeyValue(randomKey, family, qualify, Long.MAX_VALUE, Type.Put, randomKey);
+    seekKey = expectedKey.createKeyOnly(false);
+    kv = KeyValueUtil.ensureKeyValue(cachedMobFile.readCell(seekKey, false));
+    MobTestUtil.assertKeyValuesEquals(expectedKey, kv);
+
+    // Test the key which is less than the start key
+    byte[] lowerKey = Bytes.toBytes("a1"); // Smaller than "aa"
+    expectedKey = new KeyValue(startKey, family, qualify, Long.MAX_VALUE, Type.Put, startKey);
+    seekKey = new KeyValue(lowerKey, family, qualify, Long.MAX_VALUE, Type.Put, lowerKey);
+    kv = KeyValueUtil.ensureKeyValue(cachedMobFile.readCell(seekKey, false));
+    MobTestUtil.assertKeyValuesEquals(expectedKey, kv);
+
+    // Test the key which is more than the end key
+    byte[] upperKey = Bytes.toBytes("z{"); // Bigger than "zz"
+    seekKey = new KeyValue(upperKey, family, qualify, Long.MAX_VALUE, Type.Put, upperKey);
+    kv = KeyValueUtil.ensureKeyValue(cachedMobFile.readCell(seekKey, false));
+    Assert.assertNull(kv);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/5c14f749/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestDefaultMobStoreFlusher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestDefaultMobStoreFlusher.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestDefaultMobStoreFlusher.java
new file mode 100644
index 0000000..5d85718
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestDefaultMobStoreFlusher.java
@@ -0,0 +1,193 @@
+/**
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.hbase.mob;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.LargeTests;
+import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestDefaultMobStoreFlusher {
+
+ private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private final static byte [] row1 = Bytes.toBytes("row1");
+ private final static byte [] row2 = Bytes.toBytes("row2");
+ private final static byte [] family = Bytes.toBytes("family");
+ private final static byte [] qf1 = Bytes.toBytes("qf1");
+ private final static byte [] qf2 = Bytes.toBytes("qf2");
+ private final static byte [] value1 = Bytes.toBytes("value1");
+ private final static byte [] value2 = Bytes.toBytes("value2");
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+   TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
+   TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
+
+   TEST_UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+   TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testFlushNonMobFile() throws InterruptedException {
+   String TN = "testFlushNonMobFile";
+   HTable table = null;
+   HBaseAdmin admin = null;
+
+   try {
+     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(TN));
+     HColumnDescriptor hcd = new HColumnDescriptor(family);
+     hcd.setMaxVersions(4);
+     desc.addFamily(hcd);
+
+     admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+     admin.createTable(desc);
+     table = new HTable(TEST_UTIL.getConfiguration(), TN);
+
+     //Put data
+     Put put0 = new Put(row1);
+     put0.add(family, qf1, 1, value1);
+     table.put(put0);
+
+     //Put more data
+     Put put1 = new Put(row2);
+     put1.add(family, qf2, 1, value2);
+     table.put(put1);
+
+     //Flush
+     table.flushCommits();
+     admin.flush(TN);
+
+     Scan scan = new Scan();
+     scan.addColumn(family, qf1);
+     scan.setMaxVersions(4);
+     ResultScanner scanner = table.getScanner(scan);
+
+     //Compare
+     Result result = scanner.next();
+     int size = 0;
+     while (result != null) {
+       size++;
+       List<Cell> cells = result.getColumnCells(family, qf1);
+       // Verify the cell size
+       Assert.assertEquals(1, cells.size());
+       // Verify the value
+       Assert.assertEquals(Bytes.toString(value1),
+           Bytes.toString(CellUtil.cloneValue(cells.get(0))));
+       result = scanner.next();
+     }
+     scanner.close();
+     Assert.assertEquals(1, size);
+     admin.close();
+   } catch (MasterNotRunningException e1) {
+     e1.printStackTrace();
+   } catch (ZooKeeperConnectionException e2) {
+     e2.printStackTrace();
+   } catch (IOException e3) {
+     e3.printStackTrace();
+   }
+ }
+
+ @Test
+ public void testFlushMobFile() throws InterruptedException {
+   String TN = "testFlushMobFile";
+   HTable table = null;
+   HBaseAdmin admin = null;
+
+   try {
+     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(TN));
+     HColumnDescriptor hcd = new HColumnDescriptor(family);
+     hcd.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
+     hcd.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(3L));
+     hcd.setMaxVersions(4);
+     desc.addFamily(hcd);
+
+     admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+     admin.createTable(desc);
+     table = new HTable(TEST_UTIL.getConfiguration(), TN);
+
+     //put data
+     Put put0 = new Put(row1);
+     put0.add(family, qf1, 1, value1);
+     table.put(put0);
+
+     //put more data
+     Put put1 = new Put(row2);
+     put1.add(family, qf2, 1, value2);
+     table.put(put1);
+
+     //flush
+     table.flushCommits();
+     admin.flush(TN);
+
+     //Scan
+     Scan scan = new Scan();
+     scan.addColumn(family, qf1);
+     scan.setMaxVersions(4);
+     ResultScanner scanner = table.getScanner(scan);
+
+     //Compare
+     Result result = scanner.next();
+     int size = 0;
+     while (result != null) {
+       size++;
+       List<Cell> cells = result.getColumnCells(family, qf1);
+       // Verify the the cell size
+       Assert.assertEquals(1, cells.size());
+       // Verify the value
+       Assert.assertEquals(Bytes.toString(value1),
+           Bytes.toString(CellUtil.cloneValue(cells.get(0))));
+       result = scanner.next();
+     }
+     scanner.close();
+     Assert.assertEquals(1, size);
+     admin.close();
+   } catch (MasterNotRunningException e1) {
+     e1.printStackTrace();
+   } catch (ZooKeeperConnectionException e2) {
+     e2.printStackTrace();
+   } catch (IOException e3) {
+     e3.printStackTrace();
+   }
+ }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/5c14f749/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobDataBlockEncoding.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobDataBlockEncoding.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobDataBlockEncoding.java
new file mode 100644
index 0000000..ae10aad
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobDataBlockEncoding.java
@@ -0,0 +1,141 @@
+/**
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.hbase.mob;
+
+import java.util.List;
+import java.util.Random;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(MediumTests.class)
+public class TestMobDataBlockEncoding {
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private final static byte [] row1 = Bytes.toBytes("row1");
+  private final static byte [] family = Bytes.toBytes("family");
+  private final static byte [] qf1 = Bytes.toBytes("qualifier1");
+  private final static byte [] qf2 = Bytes.toBytes("qualifier2");
+  protected final byte[] qf3 = Bytes.toBytes("qualifier3");
+  private static HTable table;
+  private static HBaseAdmin admin;
+  private static HColumnDescriptor hcd;
+  private static HTableDescriptor desc;
+  private static Random random = new Random();
+  private static long defaultThreshold = 10;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
+    TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
+
+    TEST_UTIL.startMiniCluster(1);
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  public void setUp(long threshold, String TN, DataBlockEncoding encoding)
+      throws Exception {
+    desc = new HTableDescriptor(TableName.valueOf(TN));
+    hcd = new HColumnDescriptor(family);
+    hcd.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
+    hcd.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(threshold));
+    hcd.setMaxVersions(4);
+    hcd.setDataBlockEncoding(encoding);
+    desc.addFamily(hcd);
+    admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+    admin.createTable(desc);
+    table = new HTable(TEST_UTIL.getConfiguration(), TN);
+  }
+
+  /**
+   * Generate the mob value.
+   *
+   * @param size the size of the value
+   * @return the mob value generated
+   */
+  private static byte[] generateMobValue(int size) {
+    byte[] mobVal = new byte[size];
+    random.nextBytes(mobVal);
+    return mobVal;
+  }
+
+  @Test
+  public void testDataBlockEncoding() throws Exception {
+    for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
+      testDataBlockEncoding(encoding);
+    }
+  }
+
+  public void testDataBlockEncoding(DataBlockEncoding encoding) throws Exception {
+    String TN = "testDataBlockEncoding" + encoding;
+    setUp(defaultThreshold, TN, encoding);
+    long ts1 = System.currentTimeMillis();
+    long ts2 = ts1 + 1;
+    long ts3 = ts1 + 2;
+    byte[] value = generateMobValue((int) defaultThreshold + 1);
+
+    Put put1 = new Put(row1);
+    put1.add(family, qf1, ts3, value);
+    put1.add(family, qf2, ts2, value);
+    put1.add(family, qf3, ts1, value);
+    table.put(put1);
+
+    table.flushCommits();
+    admin.flush(TN);
+
+    Scan scan = new Scan();
+    scan.setMaxVersions(4);
+
+    ResultScanner results = table.getScanner(scan);
+    int count = 0;
+    for (Result res : results) {
+      List<Cell> cells = res.listCells();
+      for(Cell cell : cells) {
+        // Verify the value
+        Assert.assertEquals(Bytes.toString(value),
+            Bytes.toString(CellUtil.cloneValue(cell)));
+        count++;
+      }
+    }
+    results.close();
+    Assert.assertEquals(3, count);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/5c14f749/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java
new file mode 100644
index 0000000..f6511f7
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFile.java
@@ -0,0 +1,124 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mob;
+
+import junit.framework.TestCase;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValue.Type;
+import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.HFileContext;
+import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestMobFile extends TestCase {
+  static final Log LOG = LogFactory.getLog(TestMobFile.class);
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private Configuration conf = TEST_UTIL.getConfiguration();
+  private CacheConfig cacheConf =  new CacheConfig(conf);
+  private final String TABLE = "tableName";
+  private final String FAMILY = "familyName";
+
+  @Test
+  public void testReadKeyValue() throws Exception {
+    FileSystem fs = FileSystem.get(conf);
+	Path testDir = FSUtils.getRootDir(conf);
+    Path outputDir = new Path(new Path(testDir, TABLE), FAMILY);
+    HFileContext meta = new HFileContextBuilder().withBlockSize(8*1024).build();
+    StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs)
+            .withOutputDir(outputDir)
+            .withFileContext(meta)
+            .build();
+    String caseName = getName();
+    MobTestUtil.writeStoreFile(writer, caseName);
+
+    MobFile mobFile = new MobFile(new StoreFile(fs, writer.getPath(),
+        conf, cacheConf, BloomType.NONE));
+    byte[] family = Bytes.toBytes(caseName);
+    byte[] qualify = Bytes.toBytes(caseName);
+
+    // Test the start key
+    byte[] startKey = Bytes.toBytes("aa");  // The start key bytes
+    KeyValue expectedKey =
+        new KeyValue(startKey, family, qualify, Long.MAX_VALUE, Type.Put, startKey);
+    KeyValue seekKey = expectedKey.createKeyOnly(false);
+    KeyValue kv = KeyValueUtil.ensureKeyValue(mobFile.readCell(seekKey, false));
+    MobTestUtil.assertKeyValuesEquals(expectedKey, kv);
+
+    // Test the end key
+    byte[] endKey = Bytes.toBytes("zz");  // The end key bytes
+    expectedKey = new KeyValue(endKey, family, qualify, Long.MAX_VALUE, Type.Put, endKey);
+    seekKey = expectedKey.createKeyOnly(false);
+    kv = KeyValueUtil.ensureKeyValue(mobFile.readCell(seekKey, false));
+    MobTestUtil.assertKeyValuesEquals(expectedKey, kv);
+
+    // Test the random key
+    byte[] randomKey = Bytes.toBytes(MobTestUtil.generateRandomString(2));
+    expectedKey = new KeyValue(randomKey, family, qualify, Long.MAX_VALUE, Type.Put, randomKey);
+    seekKey = expectedKey.createKeyOnly(false);
+    kv = KeyValueUtil.ensureKeyValue(mobFile.readCell(seekKey, false));
+    MobTestUtil.assertKeyValuesEquals(expectedKey, kv);
+
+    // Test the key which is less than the start key
+    byte[] lowerKey = Bytes.toBytes("a1"); // Smaller than "aa"
+    expectedKey = new KeyValue(startKey, family, qualify, Long.MAX_VALUE, Type.Put, startKey);
+    seekKey = new KeyValue(lowerKey, family, qualify, Long.MAX_VALUE, Type.Put, lowerKey);
+    kv = KeyValueUtil.ensureKeyValue(mobFile.readCell(seekKey, false));
+    MobTestUtil.assertKeyValuesEquals(expectedKey, kv);
+
+    // Test the key which is more than the end key
+    byte[] upperKey = Bytes.toBytes("z{"); // Bigger than "zz"
+    seekKey = new KeyValue(upperKey, family, qualify, Long.MAX_VALUE, Type.Put, upperKey);
+    kv = KeyValueUtil.ensureKeyValue(mobFile.readCell(seekKey, false));
+    assertNull(kv);
+  }
+
+  @Test
+  public void testGetScanner() throws Exception {
+    FileSystem fs = FileSystem.get(conf);
+    Path testDir = FSUtils.getRootDir(conf);
+    Path outputDir = new Path(new Path(testDir, TABLE), FAMILY);
+    HFileContext meta = new HFileContextBuilder().withBlockSize(8*1024).build();
+    StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, fs)
+            .withOutputDir(outputDir)
+            .withFileContext(meta)
+            .build();
+    MobTestUtil.writeStoreFile(writer, getName());
+
+    MobFile mobFile = new MobFile(new StoreFile(fs, writer.getPath(),
+        conf, cacheConf, BloomType.NONE));
+    assertNotNull(mobFile.getScanner());
+    assertTrue(mobFile.getScanner() instanceof StoreFileScanner);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/5c14f749/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCache.java
new file mode 100644
index 0000000..9478544
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileCache.java
@@ -0,0 +1,206 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mob;
+
+import java.io.IOException;
+import java.util.Date;
+
+import junit.framework.TestCase;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.regionserver.HMobStore;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestMobFileCache extends TestCase {
+  static final Log LOG = LogFactory.getLog(TestMobFileCache.class);
+  private HBaseTestingUtility UTIL;
+  private HRegion region;
+  private Configuration conf;
+  private MobCacheConfig mobCacheConf;
+  private MobFileCache mobFileCache;
+  private Date currentDate = new Date();
+  private final String TEST_CACHE_SIZE = "2";
+  private final int EXPECTED_CACHE_SIZE_ZERO = 0;
+  private final int EXPECTED_CACHE_SIZE_ONE = 1;
+  private final int EXPECTED_CACHE_SIZE_TWO = 2;
+  private final int EXPECTED_CACHE_SIZE_THREE = 3;
+  private final long EXPECTED_REFERENCE_ONE = 1;
+  private final long EXPECTED_REFERENCE_TWO = 2;
+
+  private final String TABLE = "tableName";
+  private final String FAMILY1 = "family1";
+  private final String FAMILY2 = "family2";
+  private final String FAMILY3 = "family3";
+
+  private final byte[] ROW = Bytes.toBytes("row");
+  private final byte[] ROW2 = Bytes.toBytes("row2");
+  private final byte[] VALUE = Bytes.toBytes("value");
+  private final byte[] VALUE2 = Bytes.toBytes("value2");
+  private final byte[] QF1 = Bytes.toBytes("qf1");
+  private final byte[] QF2 = Bytes.toBytes("qf2");
+  private final byte[] QF3 = Bytes.toBytes("qf3");
+
+  @Override
+  public void setUp() throws Exception {
+    UTIL = HBaseTestingUtility.createLocalHTU();
+    conf = UTIL.getConfiguration();
+    HTableDescriptor htd = UTIL.createTableDescriptor("testMobFileCache");
+    HColumnDescriptor hcd1 = new HColumnDescriptor(FAMILY1);
+    hcd1.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
+    hcd1.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(0L));
+    HColumnDescriptor hcd2 = new HColumnDescriptor(FAMILY2);
+    hcd2.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
+    hcd2.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(0L));
+    HColumnDescriptor hcd3 = new HColumnDescriptor(FAMILY3);
+    hcd3.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
+    hcd3.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(0L));
+    htd.addFamily(hcd1);
+    htd.addFamily(hcd2);
+    htd.addFamily(hcd3);
+    region = UTIL.createLocalHRegion(htd, null, null);
+  }
+
+  @Override
+  protected void tearDown() throws Exception {
+    region.close();
+    region.getFilesystem().delete(UTIL.getDataTestDir(), true);
+  }
+
+  /**
+   * Create the mob store file.
+   * @param family
+   */
+  private Path createMobStoreFile(String family) throws IOException {
+    return createMobStoreFile(HBaseConfiguration.create(), family);
+  }
+
+  /**
+   * Create the mob store file
+   * @param conf
+   * @param family
+   */
+  private Path createMobStoreFile(Configuration conf, String family) throws IOException {
+    HColumnDescriptor hcd = new HColumnDescriptor(family);
+    hcd.setMaxVersions(4);
+    hcd.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
+    mobCacheConf = new MobCacheConfig(conf, hcd);
+    return createMobStoreFile(conf, hcd);
+  }
+
+  /**
+   * Create the mob store file
+   * @param conf
+   * @param hcd
+   */
+  private Path createMobStoreFile(Configuration conf, HColumnDescriptor hcd)
+      throws IOException {
+    // Setting up a Store
+    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
+    htd.addFamily(hcd);
+    HMobStore mobStore = (HMobStore) region.getStore(hcd.getName());
+    KeyValue key1 = new KeyValue(ROW, hcd.getName(), QF1, 1, VALUE);
+    KeyValue key2 = new KeyValue(ROW, hcd.getName(), QF2, 1, VALUE);
+    KeyValue key3 = new KeyValue(ROW2, hcd.getName(), QF3, 1, VALUE2);
+    KeyValue[] keys = new KeyValue[] { key1, key2, key3 };
+    int maxKeyCount = keys.length;
+    HRegionInfo regionInfo = new HRegionInfo();
+    StoreFile.Writer mobWriter = mobStore.createWriterInTmp(currentDate,
+        maxKeyCount, hcd.getCompactionCompression(), regionInfo.getStartKey());
+    Path mobFilePath = mobWriter.getPath();
+    String fileName = mobFilePath.getName();
+    mobWriter.append(key1);
+    mobWriter.append(key2);
+    mobWriter.append(key3);
+    mobWriter.close();
+    String targetPathName = MobUtils.formatDate(currentDate);
+    Path targetPath = new Path(mobStore.getPath(), targetPathName);
+    mobStore.commitFile(mobFilePath, targetPath);
+    return new Path(targetPath, fileName);
+  }
+
+  @Test
+  public void testMobFileCache() throws Exception {
+    FileSystem fs = FileSystem.get(conf);
+    conf.set(MobConstants.MOB_FILE_CACHE_SIZE_KEY, TEST_CACHE_SIZE);
+    mobFileCache = new MobFileCache(conf);
+    Path file1Path = createMobStoreFile(FAMILY1);
+    Path file2Path = createMobStoreFile(FAMILY2);
+    Path file3Path = createMobStoreFile(FAMILY3);
+
+    // Before open one file by the MobFileCache
+    assertEquals(EXPECTED_CACHE_SIZE_ZERO, mobFileCache.getCacheSize());
+    // Open one file by the MobFileCache
+    CachedMobFile cachedMobFile1 = (CachedMobFile) mobFileCache.openFile(
+        fs, file1Path, mobCacheConf);
+    assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize());
+    assertNotNull(cachedMobFile1);
+    assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount());
+
+    // The evict is also managed by a schedule thread pool.
+    // And its check period is set as 3600 seconds by default.
+    // This evict should get the lock at the most time
+    mobFileCache.evict();  // Cache not full, evict it
+    assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize());
+    assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount());
+
+    mobFileCache.evictFile(file1Path.getName());  // Evict one file
+    assertEquals(EXPECTED_CACHE_SIZE_ZERO, mobFileCache.getCacheSize());
+    assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile1.getReferenceCount());
+
+    cachedMobFile1.close();  // Close the cached mob file
+
+    // Reopen three cached file
+    cachedMobFile1 = (CachedMobFile) mobFileCache.openFile(
+        fs, file1Path, mobCacheConf);
+    assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize());
+    CachedMobFile cachedMobFile2 = (CachedMobFile) mobFileCache.openFile(
+        fs, file2Path, mobCacheConf);
+    assertEquals(EXPECTED_CACHE_SIZE_TWO, mobFileCache.getCacheSize());
+    CachedMobFile cachedMobFile3 = (CachedMobFile) mobFileCache.openFile(
+        fs, file3Path, mobCacheConf);
+    // Before the evict
+    // Evict the cache, should clost the first file 1
+    assertEquals(EXPECTED_CACHE_SIZE_THREE, mobFileCache.getCacheSize());
+    assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile1.getReferenceCount());
+    assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile2.getReferenceCount());
+    assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile3.getReferenceCount());
+    mobFileCache.evict();
+    assertEquals(EXPECTED_CACHE_SIZE_ONE, mobFileCache.getCacheSize());
+    assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile1.getReferenceCount());
+    assertEquals(EXPECTED_REFERENCE_ONE, cachedMobFile2.getReferenceCount());
+    assertEquals(EXPECTED_REFERENCE_TWO, cachedMobFile3.getReferenceCount());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/5c14f749/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileName.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileName.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileName.java
new file mode 100644
index 0000000..9a6cf7f
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/TestMobFileName.java
@@ -0,0 +1,79 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mob;
+
+import java.util.Date;
+import java.util.Random;
+import java.util.UUID;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.MD5Hash;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestMobFileName extends TestCase {
+
+  private String uuid;
+  private Date date;
+  private String dateStr;
+  private byte[] startKey;
+
+  public void setUp() {
+    Random random = new Random();
+    uuid = UUID.randomUUID().toString().replaceAll("-", "");
+    date = new Date();
+    dateStr = MobUtils.formatDate(date);
+    startKey = Bytes.toBytes(random.nextInt());
+  }
+
+  @Test
+  public void testHashCode() {
+    assertEquals(MobFileName.create(startKey, dateStr, uuid).hashCode(),
+        MobFileName.create(startKey, dateStr, uuid).hashCode());
+    assertNotSame(MobFileName.create(startKey, dateStr, uuid).hashCode(),
+        MobFileName.create(startKey, dateStr, uuid).hashCode());
+  }
+
+  @Test
+  public void testCreate() {
+    MobFileName mobFileName = MobFileName.create(startKey, dateStr, uuid);
+    assertEquals(mobFileName, MobFileName.create(mobFileName.getFileName()));
+  }
+
+  @Test
+  public void testGet() {
+    MobFileName mobFileName = MobFileName.create(startKey, dateStr, uuid);
+    assertEquals(MD5Hash.getMD5AsHex(startKey, 0, startKey.length), mobFileName.getStartKey());
+    assertEquals(dateStr, mobFileName.getDate());
+    assertEquals(mobFileName.getFileName(), MD5Hash.getMD5AsHex(startKey, 0, startKey.length)
+        + dateStr + uuid);
+  }
+
+  @Test
+  public void testEquals() {
+    MobFileName mobFileName = MobFileName.create(startKey, dateStr, uuid);
+    assertTrue(mobFileName.equals(mobFileName));
+    assertFalse(mobFileName.equals(this));
+    assertTrue(mobFileName.equals(MobFileName.create(startKey, dateStr, uuid)));
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/5c14f749/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteMobTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteMobTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteMobTable.java
new file mode 100644
index 0000000..ed4b703
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDeleteMobTable.java
@@ -0,0 +1,225 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.mob.MobUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(MediumTests.class)
+public class TestDeleteMobTable {
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private final static byte[] FAMILY = Bytes.toBytes("family");
+  private final static byte[] QF = Bytes.toBytes("qualifier");
+  private static Random random = new Random();
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
+    TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
+    TEST_UTIL.startMiniCluster(1);
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * Generate the mob value.
+   *
+   * @param size
+   *          the size of the value
+   * @return the mob value generated
+   */
+  private static byte[] generateMobValue(int size) {
+    byte[] mobVal = new byte[size];
+    random.nextBytes(mobVal);
+    return mobVal;
+  }
+
+  @Test
+  public void testDeleteMobTable() throws Exception {
+    byte[] tableName = Bytes.toBytes("testDeleteMobTable");
+    TableName tn = TableName.valueOf(tableName);
+    HTableDescriptor htd = new HTableDescriptor(tn);
+    HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
+    hcd.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
+    hcd.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(0L));
+    htd.addFamily(hcd);
+    HBaseAdmin admin = null;
+    HTable table = null;
+    try {
+      admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+      admin.createTable(htd);
+      table = new HTable(TEST_UTIL.getConfiguration(), tableName);
+      byte[] value = generateMobValue(10);
+
+      byte[] row = Bytes.toBytes("row");
+      Put put = new Put(row);
+      put.add(FAMILY, QF, EnvironmentEdgeManager.currentTime(), value);
+      table.put(put);
+
+      table.flushCommits();
+      admin.flush(tableName);
+
+      // the mob file exists
+      Assert.assertEquals(1, countMobFiles(tn, hcd.getNameAsString()));
+      Assert.assertEquals(0, countArchiveMobFiles(tn, hcd.getNameAsString()));
+      String fileName = assertHasOneMobRow(table, tn, hcd.getNameAsString());
+      Assert.assertFalse(mobArchiveExist(tn, hcd.getNameAsString(), fileName));
+      Assert.assertTrue(mobTableDirExist(tn));
+      table.close();
+
+      admin.disableTable(tn);
+      admin.deleteTable(tn);
+
+      Assert.assertFalse(admin.tableExists(tn));
+      Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString()));
+      Assert.assertEquals(1, countArchiveMobFiles(tn, hcd.getNameAsString()));
+      Assert.assertTrue(mobArchiveExist(tn, hcd.getNameAsString(), fileName));
+      Assert.assertFalse(mobTableDirExist(tn));
+    } finally {
+      if (admin != null) {
+        admin.close();
+      }
+    }
+  }
+
+  @Test
+  public void testDeleteNonMobTable() throws Exception {
+    byte[] tableName = Bytes.toBytes("testDeleteNonMobTable");
+    TableName tn = TableName.valueOf(tableName);
+    HTableDescriptor htd = new HTableDescriptor(tn);
+    HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
+    htd.addFamily(hcd);
+    HBaseAdmin admin = null;
+    HTable table = null;
+    try {
+      admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+      admin.createTable(htd);
+      table = new HTable(TEST_UTIL.getConfiguration(), tableName);
+      byte[] value = generateMobValue(10);
+
+      byte[] row = Bytes.toBytes("row");
+      Put put = new Put(row);
+      put.add(FAMILY, QF, EnvironmentEdgeManager.currentTime(), value);
+      table.put(put);
+
+      table.flushCommits();
+      admin.flush(tableName);
+      table.close();
+
+      // the mob file doesn't exist
+      Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString()));
+      Assert.assertEquals(0, countArchiveMobFiles(tn, hcd.getNameAsString()));
+      Assert.assertFalse(mobTableDirExist(tn));
+
+      admin.disableTable(tn);
+      admin.deleteTable(tn);
+
+      Assert.assertFalse(admin.tableExists(tn));
+      Assert.assertEquals(0, countMobFiles(tn, hcd.getNameAsString()));
+      Assert.assertEquals(0, countArchiveMobFiles(tn, hcd.getNameAsString()));
+      Assert.assertFalse(mobTableDirExist(tn));
+    } finally {
+      if (admin != null) {
+        admin.close();
+      }
+    }
+  }
+
+  private int countMobFiles(TableName tn, String familyName) throws IOException {
+    FileSystem fs = TEST_UTIL.getTestFileSystem();
+    Path mobFileDir = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tn, familyName);
+    if (fs.exists(mobFileDir)) {
+      return fs.listStatus(mobFileDir).length;
+    } else {
+      return 0;
+    }
+  }
+
+  private int countArchiveMobFiles(TableName tn, String familyName)
+      throws IOException {
+    FileSystem fs = TEST_UTIL.getTestFileSystem();
+    Path storePath = HFileArchiveUtil.getStoreArchivePath(TEST_UTIL.getConfiguration(), tn,
+        MobUtils.getMobRegionInfo(tn).getEncodedName(), familyName);
+    if (fs.exists(storePath)) {
+      return fs.listStatus(storePath).length;
+    } else {
+      return 0;
+    }
+  }
+
+  private boolean mobTableDirExist(TableName tn) throws IOException {
+    FileSystem fs = TEST_UTIL.getTestFileSystem();
+    Path tableDir = FSUtils.getTableDir(MobUtils.getMobHome(TEST_UTIL.getConfiguration()), tn);
+    return fs.exists(tableDir);
+  }
+
+  private boolean mobArchiveExist(TableName tn, String familyName, String fileName)
+      throws IOException {
+    FileSystem fs = TEST_UTIL.getTestFileSystem();
+    Path storePath = HFileArchiveUtil.getStoreArchivePath(TEST_UTIL.getConfiguration(), tn,
+        MobUtils.getMobRegionInfo(tn).getEncodedName(), familyName);
+    return fs.exists(new Path(storePath, fileName));
+  }
+
+  private String assertHasOneMobRow(HTable table, TableName tn, String familyName)
+      throws IOException {
+    Scan scan = new Scan();
+    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
+    ResultScanner rs = table.getScanner(scan);
+    Result r = rs.next();
+    Assert.assertNotNull(r);
+    byte[] value = r.getValue(FAMILY, QF);
+    String fileName = Bytes.toString(value, Bytes.SIZEOF_LONG, value.length - Bytes.SIZEOF_LONG);
+    Path filePath = new Path(
+        MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tn, familyName), fileName);
+    FileSystem fs = TEST_UTIL.getTestFileSystem();
+    Assert.assertTrue(fs.exists(filePath));
+    r = rs.next();
+    Assert.assertNull(r);
+    return fileName;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/5c14f749/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
new file mode 100644
index 0000000..c253611
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHMobStore.java
@@ -0,0 +1,471 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.concurrent.ConcurrentSkipListSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.mob.MobUtils;
+import org.apache.hadoop.hbase.monitoring.MonitoredTask;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+import org.mockito.Mockito;
+
+@Category(MediumTests.class)
+public class TestHMobStore {
+  public static final Log LOG = LogFactory.getLog(TestHMobStore.class);
+  @Rule public TestName name = new TestName();
+
+  private HMobStore store;
+  private HRegion region;
+  private HColumnDescriptor hcd;
+  private FileSystem fs;
+  private byte [] table = Bytes.toBytes("table");
+  private byte [] family = Bytes.toBytes("family");
+  private byte [] row = Bytes.toBytes("row");
+  private byte [] row2 = Bytes.toBytes("row2");
+  private byte [] qf1 = Bytes.toBytes("qf1");
+  private byte [] qf2 = Bytes.toBytes("qf2");
+  private byte [] qf3 = Bytes.toBytes("qf3");
+  private byte [] qf4 = Bytes.toBytes("qf4");
+  private byte [] qf5 = Bytes.toBytes("qf5");
+  private byte [] qf6 = Bytes.toBytes("qf6");
+  private byte[] value = Bytes.toBytes("value");
+  private byte[] value2 = Bytes.toBytes("value2");
+  private Path mobFilePath;
+  private Date currentDate = new Date();
+  private KeyValue seekKey1;
+  private KeyValue seekKey2;
+  private KeyValue seekKey3;
+  private NavigableSet<byte[]> qualifiers =
+    new ConcurrentSkipListSet<byte[]>(Bytes.BYTES_COMPARATOR);
+  private List<Cell> expected = new ArrayList<Cell>();
+  private long id = System.currentTimeMillis();
+  private Get get = new Get(row);
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private final String DIR = TEST_UTIL.getDataTestDir("TestHMobStore").toString();
+
+  /**
+   * Setup
+   * @throws Exception
+   */
+  @Before
+  public void setUp() throws Exception {
+    qualifiers.add(qf1);
+    qualifiers.add(qf3);
+    qualifiers.add(qf5);
+
+    Iterator<byte[]> iter = qualifiers.iterator();
+    while(iter.hasNext()){
+      byte [] next = iter.next();
+      expected.add(new KeyValue(row, family, next, 1, value));
+      get.addColumn(family, next);
+      get.setMaxVersions(); // all versions.
+    }
+  }
+
+  private void init(String methodName, Configuration conf, boolean testStore)
+  throws IOException {
+    hcd = new HColumnDescriptor(family);
+    hcd.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
+    hcd.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(3L));
+    hcd.setMaxVersions(4);
+    init(methodName, conf, hcd, testStore);
+  }
+
+  private void init(String methodName, Configuration conf,
+      HColumnDescriptor hcd, boolean testStore) throws IOException {
+    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
+    init(methodName, conf, htd, hcd, testStore);
+  }
+
+  private void init(String methodName, Configuration conf, HTableDescriptor htd,
+      HColumnDescriptor hcd, boolean testStore) throws IOException {
+    //Setting up tje Region and Store
+    Path basedir = new Path(DIR+methodName);
+    Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
+    String logName = "logs";
+    Path logdir = new Path(basedir, logName);
+    FileSystem fs = FileSystem.get(conf);
+    fs.delete(logdir, true);
+
+    htd.addFamily(hcd);
+    HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
+    HLog hlog = HLogFactory.createHLog(fs, basedir, logName, conf);
+    region = new HRegion(tableDir, hlog, fs, conf, info, htd, null);
+    store = new HMobStore(region, hcd, conf);
+    if(testStore) {
+      init(conf, hcd);
+    }
+  }
+
+  private void init(Configuration conf, HColumnDescriptor hcd)
+      throws IOException {
+    Path basedir = FSUtils.getRootDir(conf);
+    fs = FileSystem.get(conf);
+    Path homePath = new Path(basedir, Bytes.toString(family) + Path.SEPARATOR
+        + Bytes.toString(family));
+    fs.mkdirs(homePath);
+
+    KeyValue key1 = new KeyValue(row, family, qf1, 1, value);
+    KeyValue key2 = new KeyValue(row, family, qf2, 1, value);
+    KeyValue key3 = new KeyValue(row2, family, qf3, 1, value2);
+    KeyValue[] keys = new KeyValue[] { key1, key2, key3 };
+    int maxKeyCount = keys.length;
+    StoreFile.Writer mobWriter = store.createWriterInTmp(currentDate,
+        maxKeyCount, hcd.getCompactionCompression(), region.getStartKey());
+    mobFilePath = mobWriter.getPath();
+
+    mobWriter.append(key1);
+    mobWriter.append(key2);
+    mobWriter.append(key3);
+    mobWriter.close();
+
+    long valueLength1 = key1.getValueLength();
+    long valueLength2 = key2.getValueLength();
+    long valueLength3 = key3.getValueLength();
+
+    String targetPathName = MobUtils.formatDate(currentDate);
+    byte[] referenceValue =
+            Bytes.toBytes(targetPathName + Path.SEPARATOR
+                + mobFilePath.getName());
+    byte[] newReferenceValue1 = Bytes.add(Bytes.toBytes(valueLength1), referenceValue);
+    byte[] newReferenceValue2 = Bytes.add(Bytes.toBytes(valueLength2), referenceValue);
+    byte[] newReferenceValue3 = Bytes.add(Bytes.toBytes(valueLength3), referenceValue);
+    seekKey1 = new KeyValue(row, family, qf1, Long.MAX_VALUE, newReferenceValue1);
+    seekKey2 = new KeyValue(row, family, qf2, Long.MAX_VALUE, newReferenceValue2);
+    seekKey3 = new KeyValue(row2, family, qf3, Long.MAX_VALUE, newReferenceValue3);
+  }
+
+  /**
+   * Getting data from memstore
+   * @throws IOException
+   */
+  @Test
+  public void testGetFromMemStore() throws IOException {
+    final Configuration conf = HBaseConfiguration.create();
+    init(name.getMethodName(), conf, false);
+
+    //Put data in memstore
+    this.store.add(new KeyValue(row, family, qf1, 1, value));
+    this.store.add(new KeyValue(row, family, qf2, 1, value));
+    this.store.add(new KeyValue(row, family, qf3, 1, value));
+    this.store.add(new KeyValue(row, family, qf4, 1, value));
+    this.store.add(new KeyValue(row, family, qf5, 1, value));
+    this.store.add(new KeyValue(row, family, qf6, 1, value));
+
+    Scan scan = new Scan(get);
+    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
+        scan.getFamilyMap().get(store.getFamily().getName()),
+        0);
+
+    List<Cell> results = new ArrayList<Cell>();
+    scanner.next(results);
+    Collections.sort(results, KeyValue.COMPARATOR);
+    scanner.close();
+
+    //Compare
+    Assert.assertEquals(expected.size(), results.size());
+    for(int i=0; i<results.size(); i++) {
+      // Verify the values
+      Assert.assertEquals(expected.get(i), results.get(i));
+    }
+  }
+
+  /**
+   * Getting MOB data from files
+   * @throws IOException
+   */
+  @Test
+  public void testGetFromFiles() throws IOException {
+    final Configuration conf = TEST_UTIL.getConfiguration();
+    init(name.getMethodName(), conf, false);
+
+    //Put data in memstore
+    this.store.add(new KeyValue(row, family, qf1, 1, value));
+    this.store.add(new KeyValue(row, family, qf2, 1, value));
+    //flush
+    flush(1);
+
+    //Add more data
+    this.store.add(new KeyValue(row, family, qf3, 1, value));
+    this.store.add(new KeyValue(row, family, qf4, 1, value));
+    //flush
+    flush(2);
+
+    //Add more data
+    this.store.add(new KeyValue(row, family, qf5, 1, value));
+    this.store.add(new KeyValue(row, family, qf6, 1, value));
+    //flush
+    flush(3);
+
+    Scan scan = new Scan(get);
+    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
+        scan.getFamilyMap().get(store.getFamily().getName()),
+        0);
+
+    List<Cell> results = new ArrayList<Cell>();
+    scanner.next(results);
+    Collections.sort(results, KeyValue.COMPARATOR);
+    scanner.close();
+
+    //Compare
+    Assert.assertEquals(expected.size(), results.size());
+    for(int i=0; i<results.size(); i++) {
+      Assert.assertEquals(expected.get(i), results.get(i));
+    }
+  }
+
+  /**
+   * Getting the reference data from files
+   * @throws IOException
+   */
+  @Test
+  public void testGetReferencesFromFiles() throws IOException {
+    final Configuration conf = HBaseConfiguration.create();
+    init(name.getMethodName(), conf, false);
+
+    //Put data in memstore
+    this.store.add(new KeyValue(row, family, qf1, 1, value));
+    this.store.add(new KeyValue(row, family, qf2, 1, value));
+    //flush
+    flush(1);
+
+    //Add more data
+    this.store.add(new KeyValue(row, family, qf3, 1, value));
+    this.store.add(new KeyValue(row, family, qf4, 1, value));
+    //flush
+    flush(2);
+
+    //Add more data
+    this.store.add(new KeyValue(row, family, qf5, 1, value));
+    this.store.add(new KeyValue(row, family, qf6, 1, value));
+    //flush
+    flush(3);
+
+    Scan scan = new Scan(get);
+    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
+    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
+      scan.getFamilyMap().get(store.getFamily().getName()),
+      0);
+
+    List<Cell> results = new ArrayList<Cell>();
+    scanner.next(results);
+    Collections.sort(results, KeyValue.COMPARATOR);
+    scanner.close();
+
+    //Compare
+    Assert.assertEquals(expected.size(), results.size());
+    for(int i=0; i<results.size(); i++) {
+      Cell cell = results.get(i);
+      Assert.assertTrue(MobUtils.isMobReferenceCell(cell));
+    }
+  }
+
+  /**
+   * Getting data from memstore and files
+   * @throws IOException
+   */
+  @Test
+  public void testGetFromMemStoreAndFiles() throws IOException {
+
+    final Configuration conf = HBaseConfiguration.create();
+
+    init(name.getMethodName(), conf, false);
+
+    //Put data in memstore
+    this.store.add(new KeyValue(row, family, qf1, 1, value));
+    this.store.add(new KeyValue(row, family, qf2, 1, value));
+    //flush
+    flush(1);
+
+    //Add more data
+    this.store.add(new KeyValue(row, family, qf3, 1, value));
+    this.store.add(new KeyValue(row, family, qf4, 1, value));
+    //flush
+    flush(2);
+
+    //Add more data
+    this.store.add(new KeyValue(row, family, qf5, 1, value));
+    this.store.add(new KeyValue(row, family, qf6, 1, value));
+
+    Scan scan = new Scan(get);
+    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
+        scan.getFamilyMap().get(store.getFamily().getName()),
+        0);
+
+    List<Cell> results = new ArrayList<Cell>();
+    scanner.next(results);
+    Collections.sort(results, KeyValue.COMPARATOR);
+    scanner.close();
+
+    //Compare
+    Assert.assertEquals(expected.size(), results.size());
+    for(int i=0; i<results.size(); i++) {
+      Assert.assertEquals(expected.get(i), results.get(i));
+    }
+  }
+
+  /**
+   * Getting data from memstore and files
+   * @throws IOException
+   */
+  @Test
+  public void testMobCellSizeThreshold() throws IOException {
+
+    final Configuration conf = HBaseConfiguration.create();
+
+    HColumnDescriptor hcd;
+    hcd = new HColumnDescriptor(family);
+    hcd.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
+    hcd.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(100L));
+    hcd.setMaxVersions(4);
+    init(name.getMethodName(), conf, hcd, false);
+
+    //Put data in memstore
+    this.store.add(new KeyValue(row, family, qf1, 1, value));
+    this.store.add(new KeyValue(row, family, qf2, 1, value));
+    //flush
+    flush(1);
+
+    //Add more data
+    this.store.add(new KeyValue(row, family, qf3, 1, value));
+    this.store.add(new KeyValue(row, family, qf4, 1, value));
+    //flush
+    flush(2);
+
+    //Add more data
+    this.store.add(new KeyValue(row, family, qf5, 1, value));
+    this.store.add(new KeyValue(row, family, qf6, 1, value));
+    //flush
+    flush(3);
+
+    Scan scan = new Scan(get);
+    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
+    InternalScanner scanner = (InternalScanner) store.getScanner(scan,
+      scan.getFamilyMap().get(store.getFamily().getName()),
+      0);
+
+    List<Cell> results = new ArrayList<Cell>();
+    scanner.next(results);
+    Collections.sort(results, KeyValue.COMPARATOR);
+    scanner.close();
+
+    //Compare
+    Assert.assertEquals(expected.size(), results.size());
+    for(int i=0; i<results.size(); i++) {
+      Cell cell = results.get(i);
+      //this is not mob reference cell.
+      Assert.assertFalse(MobUtils.isMobReferenceCell(cell));
+      Assert.assertEquals(expected.get(i), results.get(i));
+      Assert.assertEquals(100, MobUtils.getMobThreshold(store.getFamily()));
+    }
+  }
+
+  @Test
+  public void testCommitFile() throws Exception {
+    final Configuration conf = HBaseConfiguration.create();
+    init(name.getMethodName(), conf, true);
+    String targetPathName = MobUtils.formatDate(new Date());
+    Path targetPath = new Path(store.getPath(), (targetPathName
+        + Path.SEPARATOR + mobFilePath.getName()));
+    fs.delete(targetPath, true);
+    Assert.assertFalse(fs.exists(targetPath));
+    //commit file
+    store.commitFile(mobFilePath, targetPath);
+    Assert.assertTrue(fs.exists(targetPath));
+  }
+
+  @Test
+  public void testResolve() throws Exception {
+    final Configuration conf = HBaseConfiguration.create();
+    init(name.getMethodName(), conf, true);
+    String targetPathName = MobUtils.formatDate(currentDate);
+    Path targetPath = new Path(store.getPath(), targetPathName);
+    store.commitFile(mobFilePath, targetPath);
+    //resolve
+    Cell resultCell1 = store.resolve(seekKey1, false);
+    Cell resultCell2 = store.resolve(seekKey2, false);
+    Cell resultCell3 = store.resolve(seekKey3, false);
+    //compare
+    Assert.assertEquals(Bytes.toString(value),
+        Bytes.toString(CellUtil.cloneValue(resultCell1)));
+    Assert.assertEquals(Bytes.toString(value),
+        Bytes.toString(CellUtil.cloneValue(resultCell2)));
+    Assert.assertEquals(Bytes.toString(value2),
+        Bytes.toString(CellUtil.cloneValue(resultCell3)));
+  }
+
+  /**
+   * Flush the memstore
+   * @param storeFilesSize
+   * @throws IOException
+   */
+  private void flush(int storeFilesSize) throws IOException{
+    this.store.snapshot();
+    flushStore(store, id++);
+    Assert.assertEquals(storeFilesSize, this.store.getStorefiles().size());
+    Assert.assertEquals(0, ((DefaultMemStore)this.store.memstore).kvset.size());
+  }
+
+  /**
+   * Flush the memstore
+   * @param store
+   * @param id
+   * @throws IOException
+   */
+  private static void flushStore(HMobStore store, long id) throws IOException {
+    StoreFlushContext storeFlushCtx = store.createFlushContext(id);
+    storeFlushCtx.prepare();
+    storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class));
+    storeFlushCtx.commit(Mockito.mock(MonitoredTask.class));
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/5c14f749/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java
new file mode 100644
index 0000000..2af97c3
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMobStoreScanner.java
@@ -0,0 +1,318 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.mob.MobConstants;
+import org.apache.hadoop.hbase.mob.MobUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(MediumTests.class)
+public class TestMobStoreScanner {
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private final static byte [] row1 = Bytes.toBytes("row1");
+  private final static byte [] family = Bytes.toBytes("family");
+  private final static byte [] qf1 = Bytes.toBytes("qualifier1");
+  private final static byte [] qf2 = Bytes.toBytes("qualifier2");
+  protected final byte[] qf3 = Bytes.toBytes("qualifier3");
+  private static HTable table;
+  private static HBaseAdmin admin;
+  private static HColumnDescriptor hcd;
+  private static HTableDescriptor desc;
+  private static Random random = new Random();
+  private static long defaultThreshold = 10;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
+    TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
+
+    TEST_UTIL.startMiniCluster(1);
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  public void setUp(long threshold, String TN) throws Exception {
+    desc = new HTableDescriptor(TableName.valueOf(TN));
+    hcd = new HColumnDescriptor(family);
+    hcd.setValue(MobConstants.IS_MOB, Bytes.toBytes(Boolean.TRUE));
+    hcd.setValue(MobConstants.MOB_THRESHOLD, Bytes.toBytes(threshold));
+    hcd.setMaxVersions(4);
+    desc.addFamily(hcd);
+    admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+    admin.createTable(desc);
+    table = new HTable(TEST_UTIL.getConfiguration(), TN);
+  }
+
+  /**
+   * Generate the mob value.
+   *
+   * @param size the size of the value
+   * @return the mob value generated
+   */
+  private static byte[] generateMobValue(int size) {
+    byte[] mobVal = new byte[size];
+    random.nextBytes(mobVal);
+    return mobVal;
+  }
+
+  /**
+   * Set the scan attribute
+   *
+   * @param reversed if true, scan will be backward order
+   * @param mobScanRaw if true, scan will get the mob reference
+   * @return this
+   */
+  public void setScan(Scan scan, boolean reversed, boolean mobScanRaw) {
+    scan.setReversed(reversed);
+    scan.setMaxVersions(4);
+    if(mobScanRaw) {
+      scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
+    }
+  }
+
+  @Test
+  public void testMobStoreScanner() throws Exception {
+	  testGetFromFiles(false);
+	  testGetFromMemStore(false);
+    testGetReferences(false);
+    testMobThreshold(false);
+  }
+
+  @Test
+  public void testReversedMobStoreScanner() throws Exception {
+	  testGetFromFiles(true);
+	  testGetFromMemStore(true);
+    testGetReferences(true);
+    testMobThreshold(true);
+  }
+
+  public void testGetFromFiles(boolean reversed) throws Exception {
+    String TN = "testGetFromFiles" + reversed;
+    setUp(defaultThreshold, TN);
+    long ts1 = System.currentTimeMillis();
+    long ts2 = ts1 + 1;
+    long ts3 = ts1 + 2;
+    byte [] value = generateMobValue((int)defaultThreshold+1);
+
+    Put put1 = new Put(row1);
+    put1.add(family, qf1, ts3, value);
+    put1.add(family, qf2, ts2, value);
+    put1.add(family, qf3, ts1, value);
+    table.put(put1);
+
+    table.flushCommits();
+    admin.flush(TN);
+
+    Scan scan = new Scan();
+    setScan(scan, reversed, false);
+
+    ResultScanner results = table.getScanner(scan);
+    int count = 0;
+    for (Result res : results) {
+      List<Cell> cells = res.listCells();
+      for(Cell cell : cells) {
+        // Verify the value
+        Assert.assertEquals(Bytes.toString(value),
+            Bytes.toString(CellUtil.cloneValue(cell)));
+        count++;
+      }
+    }
+    results.close();
+    Assert.assertEquals(3, count);
+  }
+
+  public void testGetFromMemStore(boolean reversed) throws Exception {
+    String TN = "testGetFromMemStore" + reversed;
+    setUp(defaultThreshold, TN);
+    long ts1 = System.currentTimeMillis();
+    long ts2 = ts1 + 1;
+    long ts3 = ts1 + 2;
+    byte [] value = generateMobValue((int)defaultThreshold+1);;
+
+    Put put1 = new Put(row1);
+    put1.add(family, qf1, ts3, value);
+    put1.add(family, qf2, ts2, value);
+    put1.add(family, qf3, ts1, value);
+    table.put(put1);
+
+    Scan scan = new Scan();
+    setScan(scan, reversed, false);
+
+    ResultScanner results = table.getScanner(scan);
+    int count = 0;
+    for (Result res : results) {
+      List<Cell> cells = res.listCells();
+      for(Cell cell : cells) {
+        // Verify the value
+        Assert.assertEquals(Bytes.toString(value),
+            Bytes.toString(CellUtil.cloneValue(cell)));
+        count++;
+      }
+    }
+    results.close();
+    Assert.assertEquals(3, count);
+  }
+
+  public void testGetReferences(boolean reversed) throws Exception {
+    String TN = "testGetReferences" + reversed;
+    setUp(defaultThreshold, TN);
+    long ts1 = System.currentTimeMillis();
+    long ts2 = ts1 + 1;
+    long ts3 = ts1 + 2;
+    byte [] value = generateMobValue((int)defaultThreshold+1);;
+
+    Put put1 = new Put(row1);
+    put1.add(family, qf1, ts3, value);
+    put1.add(family, qf2, ts2, value);
+    put1.add(family, qf3, ts1, value);
+    table.put(put1);
+
+    table.flushCommits();
+    admin.flush(TN);
+
+    Scan scan = new Scan();
+    setScan(scan, reversed, true);
+
+    ResultScanner results = table.getScanner(scan);
+    int count = 0;
+    for (Result res : results) {
+      List<Cell> cells = res.listCells();
+      for(Cell cell : cells) {
+        // Verify the value
+        assertIsMobReference(cell, row1, family, value, TN);
+        count++;
+      }
+    }
+    results.close();
+    Assert.assertEquals(3, count);
+  }
+
+  public void testMobThreshold(boolean reversed) throws Exception {
+    String TN = "testMobThreshold" + reversed;
+    setUp(defaultThreshold, TN);
+    byte [] valueLess = generateMobValue((int)defaultThreshold-1);
+    byte [] valueEqual = generateMobValue((int)defaultThreshold);
+    byte [] valueGreater = generateMobValue((int)defaultThreshold+1);
+    long ts1 = System.currentTimeMillis();
+    long ts2 = ts1 + 1;
+    long ts3 = ts1 + 2;
+
+    Put put1 = new Put(row1);
+    put1.add(family, qf1, ts3, valueLess);
+    put1.add(family, qf2, ts2, valueEqual);
+    put1.add(family, qf3, ts1, valueGreater);
+    table.put(put1);
+
+    table.flushCommits();
+    admin.flush(TN);
+
+    Scan scan = new Scan();
+    setScan(scan, reversed, true);
+
+    Cell cellLess= null;
+    Cell cellEqual = null;
+    Cell cellGreater = null;
+    ResultScanner results = table.getScanner(scan);
+    int count = 0;
+    for (Result res : results) {
+      List<Cell> cells = res.listCells();
+      for(Cell cell : cells) {
+        // Verify the value
+        String qf = Bytes.toString(CellUtil.cloneQualifier(cell));
+        if(qf.equals(Bytes.toString(qf1))) {
+          cellLess = cell;
+        }
+        if(qf.equals(Bytes.toString(qf2))) {
+          cellEqual = cell;
+        }
+        if(qf.equals(Bytes.toString(qf3))) {
+          cellGreater = cell;
+        }
+        count++;
+      }
+    }
+    Assert.assertEquals(3, count);
+    assertNotMobReference(cellLess, row1, family, valueLess);
+    assertNotMobReference(cellEqual, row1, family, valueEqual);
+    assertIsMobReference(cellGreater, row1, family, valueGreater, TN);
+    results.close();
+  }
+
+  /**
+   * Assert the value is not store in mob.
+   */
+  private static void assertNotMobReference(Cell cell, byte[] row, byte[] family,
+      byte[] value) throws IOException {
+    Assert.assertEquals(Bytes.toString(row),
+        Bytes.toString(CellUtil.cloneRow(cell)));
+    Assert.assertEquals(Bytes.toString(family),
+        Bytes.toString(CellUtil.cloneFamily(cell)));
+    Assert.assertTrue(Bytes.toString(value).equals(
+        Bytes.toString(CellUtil.cloneValue(cell))));
+  }
+
+  /**
+   * Assert the value is store in mob.
+   */
+  private static void assertIsMobReference(Cell cell, byte[] row, byte[] family,
+      byte[] value, String TN) throws IOException {
+    Assert.assertEquals(Bytes.toString(row),
+        Bytes.toString(CellUtil.cloneRow(cell)));
+    Assert.assertEquals(Bytes.toString(family),
+        Bytes.toString(CellUtil.cloneFamily(cell)));
+    Assert.assertFalse(Bytes.toString(value).equals(
+        Bytes.toString(CellUtil.cloneValue(cell))));
+    byte[] referenceValue = CellUtil.cloneValue(cell);
+    String fileName = Bytes.toString(referenceValue, 8, referenceValue.length-8);
+    Path mobFamilyPath;
+    mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(),
+        TableName.valueOf(TN)), hcd.getNameAsString());
+    Path targetPath = new Path(mobFamilyPath, fileName);
+    FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
+    Assert.assertTrue(fs.exists(targetPath));
+  }
+}


Mime
View raw message