hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ramkris...@apache.org
Subject hbase git commit: HBASE-13933 - DBE's seekBefore with tags corrupts the tag's offset information thus leading to incorrect results(Ram)
Date Mon, 22 Jun 2015 05:41:42 GMT
Repository: hbase
Updated Branches:
  refs/heads/0.98 bdaf1e326 -> 213458b77


HBASE-13933 - DBE's seekBefore with tags corrupts the tag's offset
information thus leading to incorrect results(Ram)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/213458b7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/213458b7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/213458b7

Branch: refs/heads/0.98
Commit: 213458b771fb6b91357034c826cc6030ccc0e02d
Parents: bdaf1e3
Author: ramkrishna <ramkrishna.s.vasudevan@gmail.com>
Authored: Mon Jun 22 11:10:53 2015 +0530
Committer: ramkrishna <ramkrishna.s.vasudevan@gmail.com>
Committed: Mon Jun 22 11:10:53 2015 +0530

----------------------------------------------------------------------
 .../io/encoding/BufferedDataBlockEncoder.java   |  2 +
 .../hadoop/hbase/io/hfile/TestSeekTo.java       | 95 ++++++++++++++++++--
 2 files changed, 90 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/213458b7/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
index 445b897..3958c7c 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
@@ -151,6 +151,8 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
       lastCommonPrefix = nextState.lastCommonPrefix;
       nextKvOffset = nextState.nextKvOffset;
       memstoreTS = nextState.memstoreTS;
+      tagsOffset = nextState.tagsOffset;
+      tagsLength = nextState.tagsLength;
     }
 
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/213458b7/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
index c407635..9149b2a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
@@ -19,25 +19,69 @@
 package org.apache.hadoop.hbase.io.hfile;
 
 import java.io.IOException;
-
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseTestCase;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Before;
+import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
 
 /**
  * Test {@link HFileScanner#seekTo(byte[])} and its variants.
  */
 @Category(SmallTests.class)
-public class TestSeekTo extends HBaseTestCase {
+@RunWith(Parameterized.class)
+public class TestSeekTo {
+
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private final DataBlockEncoding encoding;
+
+  @Parameters
+  public static Collection<Object[]> parameters() {
+    List<Object[]> paramList = new ArrayList<Object[]>();
+    for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
+      // Remove after HBASE-13939
+      if (encoding != DataBlockEncoding.PREFIX_TREE) {
+        paramList.add(new Object[] { encoding });
+      }
+    }
+    return paramList;
+  }
 
   static boolean switchKVs = false;
 
+  public TestSeekTo(DataBlockEncoding encoding) {
+    this.encoding = encoding;
+  }
+
+  @Before
+  public void setUp() {
+    // reset
+    switchKVs = false;
+     }
+
+
   static KeyValue toKV(String row, TagUsage tagUsage) {
     if (tagUsage == TagUsage.NO_TAG) {
       return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes.toBytes("qualifier"),
@@ -52,7 +96,7 @@ public class TestSeekTo extends HBaseTestCase {
       if (!switchKVs) {
         switchKVs = true;
         return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"),
-            Bytes.toBytes("qualifier"), Bytes.toBytes("value"));
+            Bytes.toBytes("qualifier"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value"));
       } else {
         switchKVs = false;
         Tag t = new Tag((byte) 1, "myTag1");
@@ -68,15 +112,17 @@ public class TestSeekTo extends HBaseTestCase {
   }
 
   Path makeNewFile(TagUsage tagUsage) throws IOException {
-    Path ncTFile = new Path(this.testDir, "basic.hfile");
+    Path ncTFile = new Path(TEST_UTIL.getDataTestDir(), "basic.hfile");
+    FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile);
+    Configuration conf = TEST_UTIL.getConfiguration();
     if (tagUsage != TagUsage.NO_TAG) {
       conf.setInt("hfile.format.version", 3);
     } else {
       conf.setInt("hfile.format.version", 2);
     }
-    FSDataOutputStream fout = this.fs.create(ncTFile);
     int blocksize = toKV("a", tagUsage).getLength() * 3;
     HFileContext context = new HFileContextBuilder().withBlockSize(blocksize)
+        .withDataBlockEncoding(encoding)
         .withIncludesTags(true).build();
     HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout)
         .withFileContext(context)
@@ -96,6 +142,7 @@ public class TestSeekTo extends HBaseTestCase {
     return ncTFile;
   }
 
+  @Test
   public void testSeekBefore() throws Exception {
     testSeekBeforeInternals(TagUsage.NO_TAG);
     testSeekBeforeInternals(TagUsage.ONLY_TAG);
@@ -104,6 +151,8 @@ public class TestSeekTo extends HBaseTestCase {
 
   protected void testSeekBeforeInternals(TagUsage tagUsage) throws IOException {
     Path p = makeNewFile(tagUsage);
+    FileSystem fs = TEST_UTIL.getTestFileSystem();
+    Configuration conf = TEST_UTIL.getConfiguration();
     HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
     reader.loadFileInfo();
     HFileScanner scanner = reader.getScanner(false, true);
@@ -129,6 +178,15 @@ public class TestSeekTo extends HBaseTestCase {
     assertEquals("g", toRowStr(scanner.getKeyValue()));
     assertEquals(true, scanner.seekBefore(toKV("j", tagUsage).getKey()));
     assertEquals("i", toRowStr(scanner.getKeyValue()));
+    Cell cell = scanner.getKeyValue();
+    if (tagUsage != TagUsage.NO_TAG && cell.getTagsLength() > 0) {
+      Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
+          cell.getTagsLength());
+      while (tagsIterator.hasNext()) {
+        Tag next = tagsIterator.next();
+        assertEquals("myTag1", Bytes.toString(next.getValue()));
+      }
+    }
     assertEquals(true, scanner.seekBefore(toKV("k", tagUsage).getKey()));
     assertEquals("i", toRowStr(scanner.getKeyValue()));
     assertEquals(true, scanner.seekBefore(toKV("l", tagUsage).getKey()));
@@ -137,6 +195,7 @@ public class TestSeekTo extends HBaseTestCase {
     reader.close();
   }
 
+  @Test
   public void testSeekBeforeWithReSeekTo() throws Exception {
     testSeekBeforeWithReSeekToInternals(TagUsage.NO_TAG);
     testSeekBeforeWithReSeekToInternals(TagUsage.ONLY_TAG);
@@ -145,6 +204,8 @@ public class TestSeekTo extends HBaseTestCase {
 
   protected void testSeekBeforeWithReSeekToInternals(TagUsage tagUsage) throws IOException
{
     Path p = makeNewFile(tagUsage);
+    FileSystem fs = TEST_UTIL.getTestFileSystem();
+    Configuration conf = TEST_UTIL.getConfiguration();
     HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
     reader.loadFileInfo();
     HFileScanner scanner = reader.getScanner(false, true);
@@ -224,8 +285,16 @@ public class TestSeekTo extends HBaseTestCase {
     // reseekTo k
     assertEquals(0, scanner.reseekTo(toKV("k", tagUsage).getKey()));
     assertEquals("k", toRowStr(scanner.getKeyValue()));
+    deleteTestDir(fs);
   }
 
+  protected void deleteTestDir(FileSystem fs) throws IOException {
+    Path dataTestDir = TEST_UTIL.getDataTestDir();
+    if (fs.exists(dataTestDir)) {
+      fs.delete(dataTestDir, true);
+    }
+       }
+  @Test
   public void testSeekTo() throws Exception {
     testSeekToInternals(TagUsage.NO_TAG);
     testSeekToInternals(TagUsage.ONLY_TAG);
@@ -234,6 +303,8 @@ public class TestSeekTo extends HBaseTestCase {
 
   protected void testSeekToInternals(TagUsage tagUsage) throws IOException {
     Path p = makeNewFile(tagUsage);
+    FileSystem fs = TEST_UTIL.getTestFileSystem();
+    Configuration conf = TEST_UTIL.getConfiguration();
     HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
     reader.loadFileInfo();
     assertEquals(2, reader.getDataBlockIndexReader().getRootBlockCount());
@@ -249,10 +320,17 @@ public class TestSeekTo extends HBaseTestCase {
     assertEquals("g", toRowStr(scanner.getKeyValue()));
 
     assertEquals(1, scanner.seekTo(toKV("l", tagUsage).getKey()));
-    assertEquals("k", toRowStr(scanner.getKeyValue()));
+    if (encoding == DataBlockEncoding.PREFIX_TREE) {
+      // TODO : Fix this
+      assertEquals(null, scanner.getKeyValue());
+    } else {
+      assertEquals("k", toRowStr(scanner.getKeyValue()));
+    }
 
     reader.close();
   }
+
+  @Test
   public void testBlockContainingKey() throws Exception {
     testBlockContainingKeyInternals(TagUsage.NO_TAG);
     testBlockContainingKeyInternals(TagUsage.ONLY_TAG);
@@ -261,6 +339,8 @@ public class TestSeekTo extends HBaseTestCase {
 
   protected void testBlockContainingKeyInternals(TagUsage tagUsage) throws IOException {
     Path p = makeNewFile(tagUsage);
+    FileSystem fs = TEST_UTIL.getTestFileSystem();
+    Configuration conf = TEST_UTIL.getConfiguration();
     HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
     reader.loadFileInfo();
     HFileBlockIndex.BlockIndexReader blockIndexReader = 
@@ -290,6 +370,7 @@ public class TestSeekTo extends HBaseTestCase {
         toKV("l", tagUsage).getKey(), 0, klen));
 
     reader.close();
+    deleteTestDir(fs);
   }
 
 


Mime
View raw message