hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From l...@apache.org
Subject hbase git commit: HBASE-17020 keylen in midkey() dont computed correctly
Date Fri, 11 Nov 2016 02:44:50 GMT
Repository: hbase
Updated Branches:
  refs/heads/0.98 5f9cd8633 -> cf2cb620e


HBASE-17020 keylen in midkey() dont computed correctly

Signed-off-by: Yu Li <liyu@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cf2cb620
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cf2cb620
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cf2cb620

Branch: refs/heads/0.98
Commit: cf2cb620e6167c079add7b83efbcea1bed8dd7d3
Parents: 5f9cd86
Author: Yu Sun <sunyu1949@sina.com>
Authored: Fri Nov 11 08:09:45 2016 +0800
Committer: Yu Li <liyu@apache.org>
Committed: Fri Nov 11 08:10:21 2016 +0800

----------------------------------------------------------------------
 .../hadoop/hbase/io/hfile/HFileBlockIndex.java  |  2 +-
 .../hbase/io/hfile/TestHFileBlockIndex.java     | 69 ++++++++++++++++++++
 .../hbase/io/hfile/TestHFileWriterV2.java       | 29 ++++++++
 3 files changed, 99 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/cf2cb620/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
index bae16c0..c47dfb7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
@@ -344,7 +344,7 @@ public class HFileBlockIndex {
         int numDataBlocks = b.getInt();
         int keyRelOffset = b.getInt(Bytes.SIZEOF_INT * (midKeyEntry + 1));
         int keyLen = b.getInt(Bytes.SIZEOF_INT * (midKeyEntry + 2)) -
-            keyRelOffset;
+            keyRelOffset - SECONDARY_INDEX_ENTRY_OVERHEAD;
         int keyOffset = b.arrayOffset() +
             Bytes.SIZEOF_INT * (numDataBlocks + 2) + keyRelOffset +
             SECONDARY_INDEX_ENTRY_OVERHEAD;

http://git-wip-us.apache.org/repos/asf/hbase/blob/cf2cb620/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
index ec0ed05..f6c364a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
@@ -49,8 +49,10 @@ import org.apache.hadoop.hbase.KeyValueUtil;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.fs.HFileSystem;
 import org.apache.hadoop.hbase.io.compress.Compression;
+import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
 import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexChunk;
 import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexReader;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.junit.Before;
@@ -491,6 +493,73 @@ public class TestHFileBlockIndex {
   }
 
   /**
+  * to check if looks good when midKey on a leaf index block boundary
+  * @throws IOException
+  */
+  @Test
+ public void testMidKeyOnLeafIndexBlockBoundary() throws IOException {
+   Path hfilePath = new Path(TEST_UTIL.getDataTestDir(),
+       "hfile_for_midkey");
+   int maxChunkSize = 512;
+   conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, maxChunkSize);
+   // should open hfile.block.index.cacheonwrite
+   conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, true);
+
+   CacheConfig cacheConf = new CacheConfig(conf);
+   BlockCache blockCache = cacheConf.getBlockCache();
+   // Evict all blocks that were cached-on-write by the previous invocation.
+   blockCache.evictBlocksByHfileName(hfilePath.getName());
+   // Write the HFile
+   {
+     HFileContext meta = new HFileContextBuilder()
+                         .withBlockSize(SMALL_BLOCK_SIZE)
+                         .withCompression(Algorithm.NONE)
+                         .withDataBlockEncoding(DataBlockEncoding.NONE)
+                         .build();
+     HFile.Writer writer =
+           HFile.getWriterFactory(conf, cacheConf)
+               .withPath(fs, hfilePath)
+               .withFileContext(meta)
+               .create();
+     Random rand = new Random(19231737);
+     byte[] family = Bytes.toBytes("f");
+     byte[] qualifier = Bytes.toBytes("q");
+     int kvNumberToBeWritten = 16;
+     // the new generated hfile will contain 2 leaf-index blocks and 16 data blocks,
+     // midkey is just on the boundary of the first leaf-index block
+     for (int i = 0; i < kvNumberToBeWritten; ++i) {
+       byte[] row = TestHFileWriterV2.randomOrderedFixedLengthKey(rand, i, 30);
+
+       // Key will be interpreted by KeyValue.KEY_COMPARATOR
+       KeyValue kv =
+             new KeyValue(row, family, qualifier,  System.currentTimeMillis(),
+                 TestHFileWriterV2.randomFixedLengthValue(rand, SMALL_BLOCK_SIZE));
+       writer.append(kv);
+     }
+     writer.close();
+   }
+
+   // close hfile.block.index.cacheonwrite
+   conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, false);
+
+   // Read the HFile
+   HFile.Reader reader = HFile.createReader(fs, hfilePath, cacheConf, conf);
+
+   boolean hasArrayIndexOutOfBoundsException = false;
+   try {
+     // get the mid-key.
+     reader.midkey();
+   } catch (ArrayIndexOutOfBoundsException e) {
+     hasArrayIndexOutOfBoundsException = true;
+   } finally {
+     reader.close();
+   }
+
+   // to check if ArrayIndexOutOfBoundsException occured
+   assertFalse(hasArrayIndexOutOfBoundsException);
+ }
+
+  /**
    * Testing block index through the HFile writer/reader APIs. Allows to test
    * setting index block size through configuration, intermediate-level index
    * blocks, and caching index blocks on write.

http://git-wip-us.apache.org/repos/asf/hbase/blob/cf2cb620/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
index 19b460e..25df18c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
@@ -293,6 +293,25 @@ public class TestHFileWriterV2 {
     return keyBytes;
   }
 
+  public static byte[] randomOrderedFixedLengthKey(Random rand, int i, int suffixLength)
{
+    StringBuilder k = new StringBuilder();
+
+    // The fixed-length lexicographically increasing part of the key.
+    for (int bitIndex = 31; bitIndex >= 0; --bitIndex) {
+      if ((i & (1 << bitIndex)) == 0)
+        k.append("a");
+      else
+        k.append("b");
+    }
+
+    // A random suffix of the key.
+    for (int j = 0; j < suffixLength; ++j)
+      k.append(randomReadableChar(rand));
+
+    byte[] keyBytes = k.toString().getBytes();
+    return keyBytes;
+  }
+
   public static byte[] randomValue(Random rand) {
     StringBuilder v = new StringBuilder();
     for (int j = 0; j < 1 + rand.nextInt(2000); ++j) {
@@ -303,6 +322,16 @@ public class TestHFileWriterV2 {
     return valueBytes;
   }
 
+  public static byte[] randomFixedLengthValue(Random rand, int valueLength) {
+    StringBuilder v = new StringBuilder();
+    for (int j = 0; j < valueLength; ++j) {
+      v.append((char) (32 + rand.nextInt(95)));
+    }
+
+    byte[] valueBytes = v.toString().getBytes();
+    return valueBytes;
+  }
+
   public static final char randomReadableChar(Random rand) {
     int i = rand.nextInt(26 * 2 + 10 + 1);
     if (i < 26)


Mime
View raw message