hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject svn commit: r1178256 - in /hbase/branches/0.92: ./ src/main/java/org/apache/hadoop/hbase/io/hfile/ src/test/java/org/apache/hadoop/hbase/io/hfile/
Date Sun, 02 Oct 2011 21:18:53 GMT
Author: tedyu
Date: Sun Oct  2 21:18:52 2011
New Revision: 1178256

URL: http://svn.apache.org/viewvc?rev=1178256&view=rev
Log:
HBASE-4496  HFile V2 does not honor setCacheBlocks when scanning (Lars and Mikhail)

Modified:
    hbase/branches/0.92/CHANGES.txt
    hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
    hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
    hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
    hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
    hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
    hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
    hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java

Modified: hbase/branches/0.92/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/CHANGES.txt?rev=1178256&r1=1178255&r2=1178256&view=diff
==============================================================================
--- hbase/branches/0.92/CHANGES.txt (original)
+++ hbase/branches/0.92/CHANGES.txt Sun Oct  2 21:18:52 2011
@@ -316,6 +316,7 @@ Release 0.92.0 - Unreleased
                (Mikhail Bautin)
    HBASE-4209  The HBase hbase-daemon.sh SIGKILLs master when stopping it
                (Roman Shaposhnik)
+   HBASE-4496  HFile V2 does not honor setCacheBlocks when scanning (Lars and Mikhail)
 
   TESTS
    HBASE-4492  TestRollingRestart fails intermittently

Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java?rev=1178256&r1=1178255&r2=1178256&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java Sun Oct
 2 21:18:52 2011
@@ -263,63 +263,65 @@ public class HFile {
   public static final String CACHE_BLOCKS_ON_WRITE_KEY =
       "hbase.rs.cacheblocksonwrite";
 
-  /** An interface used by clients to open and iterate an {@link HFile}. */
-  public interface Reader extends Closeable {
+  /** An abstraction used by the block index */
+  public interface CachingBlockReader {
+    HFileBlock readBlock(long offset, long onDiskBlockSize,
+        boolean cacheBlock, final boolean pread, final boolean isCompaction)
+        throws IOException;
+  }
 
+  /** An interface used by clients to open and iterate an {@link HFile}. */
+  public interface Reader extends Closeable, CachingBlockReader {
     /**
      * Returns this reader's "name". Usually the last component of the path.
      * Needs to be constant as the file is being moved to support caching on
      * write.
      */
-     String getName();
+    String getName();
 
-     String getColumnFamilyName();
+    String getColumnFamilyName();
 
-     RawComparator<byte []> getComparator();
+    RawComparator<byte []> getComparator();
 
-     HFileScanner getScanner(boolean cacheBlocks,
+    HFileScanner getScanner(boolean cacheBlocks,
         final boolean pread, final boolean isCompaction);
 
-     ByteBuffer getMetaBlock(String metaBlockName,
+    ByteBuffer getMetaBlock(String metaBlockName,
         boolean cacheBlock) throws IOException;
 
-     HFileBlock readBlock(long offset, int onDiskBlockSize,
-         boolean cacheBlock, final boolean pread, final boolean isCompaction)
-         throws IOException;
-
-     Map<byte[], byte[]> loadFileInfo() throws IOException;
+    Map<byte[], byte[]> loadFileInfo() throws IOException;
 
-     byte[] getLastKey();
+    byte[] getLastKey();
 
-     byte[] midkey() throws IOException;
+    byte[] midkey() throws IOException;
 
-     long length();
+    long length();
 
-     long getEntries();
+    long getEntries();
 
-     byte[] getFirstKey();
+    byte[] getFirstKey();
 
-     long indexSize();
+    long indexSize();
 
-     byte[] getFirstRowKey();
+    byte[] getFirstRowKey();
 
-     byte[] getLastRowKey();
+    byte[] getLastRowKey();
 
-     FixedFileTrailer getTrailer();
+    FixedFileTrailer getTrailer();
 
-     HFileBlockIndex.BlockIndexReader getDataBlockIndexReader();
+    HFileBlockIndex.BlockIndexReader getDataBlockIndexReader();
 
-     HFileScanner getScanner(boolean cacheBlocks, boolean pread);
+    HFileScanner getScanner(boolean cacheBlocks, boolean pread);
 
-     Compression.Algorithm getCompressionAlgorithm();
+    Compression.Algorithm getCompressionAlgorithm();
 
     /**
      * Retrieves Bloom filter metadata as appropriate for each {@link HFile}
      * version. Knows nothing about how that metadata is structured.
      */
-     DataInput getBloomFilterMetadata() throws IOException;
+    DataInput getBloomFilterMetadata() throws IOException;
 
-     Path getPath();
+    Path getPath();
   }
 
   private static Reader pickReaderVersion(Path path, FSDataInputStream fsdis,

Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java?rev=1178256&r1=1178255&r2=1178256&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java Sun
Oct  2 21:18:52 2011
@@ -921,11 +921,9 @@ public class HFileBlock implements Cache
     DataInputStream nextBlockAsStream(BlockType blockType) throws IOException;
   }
 
-  /**
-   * Just the basic ability to read blocks, providing optional hints of
-   * on-disk-size and/or uncompressed size.
-   */
-  public interface BasicReader {
+  /** A full-fledged reader with iteration ability. */
+  public interface FSReader {
+
     /**
      * Reads the block at the given offset in the file with the given on-disk
      * size and uncompressed size.
@@ -939,10 +937,6 @@ public class HFileBlock implements Cache
      */
     HFileBlock readBlockData(long offset, long onDiskSize,
         int uncompressedSize, boolean pread) throws IOException;
-  }
-
-  /** A full-fledged reader with an iteration ability. */
-  public interface FSReader extends BasicReader {
 
     /**
      * Creates a block iterator over the given portion of the {@link HFile}.

Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java?rev=1178256&r1=1178255&r2=1178256&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
(original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java
Sun Oct  2 21:18:52 2011
@@ -38,6 +38,7 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.io.hfile.HFile.CachingBlockReader;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.apache.hadoop.hbase.util.CompoundBloomFilterWriter;
@@ -132,12 +133,12 @@ public class HFileBlockIndex {
     private int searchTreeLevel;
 
     /** A way to read {@link HFile} blocks at a given offset */
-    private HFileBlock.BasicReader blockReader;
+    private CachingBlockReader cachingBlockReader;
 
     public BlockIndexReader(final RawComparator<byte[]> c, final int treeLevel,
-        final HFileBlock.BasicReader blockReader) {
+        final CachingBlockReader cachingBlockReader) {
       this(c, treeLevel);
-      this.blockReader = blockReader;
+      this.cachingBlockReader = cachingBlockReader;
     }
 
     public BlockIndexReader(final RawComparator<byte[]> c, final int treeLevel)
@@ -176,7 +177,8 @@ public class HFileBlockIndex {
      * @throws IOException
      */
     public HFileBlock seekToDataBlock(final byte[] key, int keyOffset,
-        int keyLength, HFileBlock currentBlock)
+        int keyLength, HFileBlock currentBlock, boolean cacheBlocks,
+        boolean pread, boolean isCompaction)
         throws IOException {
       int rootLevelIndex = rootBlockContainingKey(key, keyOffset, keyLength);
       if (rootLevelIndex < 0 || rootLevelIndex >= blockOffsets.length) {
@@ -188,23 +190,44 @@ public class HFileBlockIndex {
       int currentOnDiskSize = blockDataSizes[rootLevelIndex];
 
       int lookupLevel = 1; // How many levels deep we are in our lookup.
-      HFileBlock block = blockReader.readBlockData(currentOffset,
-          currentOnDiskSize, -1, true);
-      if (block == null) {
-        throw new IOException("Failed to read block at offset " +
-            currentOffset + ", onDiskSize=" + currentOnDiskSize);
-      }
-      while (!block.getBlockType().equals(BlockType.DATA)) {
-        // Read the block. It may be intermediate level block, leaf level block
-        // or data block. In any case, we expect non-root index block format.
 
-        // We don't allow more than searchTreeLevel iterations of this loop.
+      HFileBlock block;
+      while (true) {
+
+        if (currentBlock != null && currentBlock.getOffset() == currentOffset)
+        {
+          // Avoid reading the same block again, even with caching turned off.
+          // This is crucial for compaction-type workload which might have
+          // caching turned off. This is like a one-block cache inside the
+          // scanner.
+          block = currentBlock;
+        } else {
+          // Call HFile's caching block reader API. We always cache index
+          // blocks, otherwise we might get terrible performance.
+          boolean shouldCache = cacheBlocks || (lookupLevel < searchTreeLevel);
+          block = cachingBlockReader.readBlock(currentOffset, currentOnDiskSize,
+              shouldCache, pread, isCompaction);
+        }
+
+        if (block == null) {
+          throw new IOException("Failed to read block at offset " +
+              currentOffset + ", onDiskSize=" + currentOnDiskSize);
+        }
+
+        // Found a data block, break the loop and check our level in the tree.
+        if (block.getBlockType().equals(BlockType.DATA)) {
+          break;
+        }
+
+        // Not a data block. This must be a leaf-level or intermediate-level
+        // index block. We don't allow going deeper than searchTreeLevel.
         if (++lookupLevel > searchTreeLevel) {
-          throw new IOException("Search Tree Level overflow: lookupLevel: "+
-              lookupLevel + " searchTreeLevel: " + searchTreeLevel);
+          throw new IOException("Search Tree Level overflow: lookupLevel="+
+              lookupLevel + ", searchTreeLevel=" + searchTreeLevel);
         }
 
-        // read to the byte buffer
+        // Locate the entry corresponding to the given key in the non-root
+        // (leaf or intermediate-level) index block.
         ByteBuffer buffer = block.getBufferWithoutHeader();
         if (!locateNonRootIndexEntry(buffer, key, keyOffset, keyLength,
             comparator)) {
@@ -216,16 +239,6 @@ public class HFileBlockIndex {
 
         currentOffset = buffer.getLong();
         currentOnDiskSize = buffer.getInt();
-
-        // Located a deeper-level block, now read it.
-        if (currentBlock != null && currentBlock.getOffset() == currentOffset)
-        {
-          // Avoid reading the same block.
-          block = currentBlock;
-        } else {
-          block = blockReader.readBlockData(currentOffset, currentOnDiskSize,
-              -1, true);
-        }
       }
 
       if (lookupLevel != searchTreeLevel) {
@@ -252,12 +265,15 @@ public class HFileBlockIndex {
         return midKey;
 
       if (midLeafBlockOffset >= 0) {
-        if (blockReader == null) {
+        if (cachingBlockReader == null) {
           throw new IOException("Have to read the middle leaf block but " +
               "no block reader available");
         }
-        HFileBlock midLeafBlock = blockReader.readBlockData(
-            midLeafBlockOffset, midLeafBlockOnDiskSize, -1, true);
+
+        // Caching, using pread, assuming this is not a compaction.
+        HFileBlock midLeafBlock = cachingBlockReader.readBlock(
+            midLeafBlockOffset, midLeafBlockOnDiskSize, true, true, false);
+
         ByteBuffer b = midLeafBlock.getBufferWithoutHeader();
         int numDataBlocks = b.getInt();
         int keyRelOffset = b.getInt(Bytes.SIZEOF_INT * (midKeyEntry + 1));

Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java?rev=1178256&r1=1178255&r2=1178256&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
(original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
Sun Oct  2 21:18:52 2011
@@ -643,7 +643,7 @@ public class HFileReaderV1 extends Abstr
   }
 
   @Override
-  public HFileBlock readBlock(long offset, int onDiskBlockSize,
+  public HFileBlock readBlock(long offset, long onDiskBlockSize,
       boolean cacheBlock, boolean pread, boolean isCompaction) {
     throw new UnsupportedOperationException();
   }

Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java?rev=1178256&r1=1178255&r2=1178256&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
(original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
Sun Oct  2 21:18:52 2011
@@ -37,8 +37,7 @@ import org.apache.hadoop.hbase.util.IdLo
 /**
  * {@link HFile} reader for version 2.
  */
-public class HFileReaderV2 extends AbstractHFileReader implements
-    HFileBlock.BasicReader {
+public class HFileReaderV2 extends AbstractHFileReader {
 
   private static final Log LOG = LogFactory.getLog(HFileReaderV2.class);
 
@@ -203,23 +202,6 @@ public class HFileReaderV2 extends Abstr
   }
 
   /**
-   * Implements the "basic block reader" API, used mainly by
-   * {@link HFileBlockIndex.BlockIndexReader} in
-   * {@link HFileBlockIndex.BlockIndexReader#seekToDataBlock(byte[], int, int,
-   * HFileBlock)} in a random-read access pattern.
-   */
-  @Override
-  public HFileBlock readBlockData(long offset, long onDiskSize,
-      int uncompressedSize, boolean pread) throws IOException {
-    if (onDiskSize >= Integer.MAX_VALUE) {
-      throw new IOException("Invalid on-disk size: " + onDiskSize);
-    }
-
-    // Assuming we are not doing a compaction.
-    return readBlock(offset, (int) onDiskSize, true, pread, false);
-  }
-
-  /**
    * Read in a file block.
    *
    * @param dataBlockOffset offset to read.
@@ -231,7 +213,7 @@ public class HFileReaderV2 extends Abstr
    * @throws IOException
    */
   @Override
-  public HFileBlock readBlock(long dataBlockOffset, int onDiskBlockSize,
+  public HFileBlock readBlock(long dataBlockOffset, long onDiskBlockSize,
       boolean cacheBlock, boolean pread, final boolean isCompaction)
       throws IOException {
     if (dataBlockIndexReader == null) {
@@ -498,9 +480,11 @@ public class HFileReaderV2 extends Abstr
      */
     private int seekTo(byte[] key, int offset, int length, boolean rewind)
         throws IOException {
-      HFileBlock seekToBlock =
-        ((HFileReaderV2) reader).getDataBlockIndexReader().seekToDataBlock(
-            key, offset, length, block);
+      HFileBlockIndex.BlockIndexReader indexReader =
+          reader.getDataBlockIndexReader();
+      HFileBlock seekToBlock = indexReader.seekToDataBlock(key, offset, length,
+          block, cacheBlocks, pread, isCompaction);
+
       if (seekToBlock == null) {
         // This happens if the key e.g. falls before the beginning of the file.
         return -1;
@@ -665,10 +649,9 @@ public class HFileReaderV2 extends Abstr
     @Override
     public boolean seekBefore(byte[] key, int offset, int length)
         throws IOException {
-      HFileReaderV2 reader2 = (HFileReaderV2) reader;
       HFileBlock seekToBlock =
-          reader2.getDataBlockIndexReader().seekToDataBlock(
-              key, offset, length, block);
+          reader.getDataBlockIndexReader().seekToDataBlock(key, offset,
+              length, block, cacheBlocks, pread, isCompaction);
       if (seekToBlock == null) {
         return false;
       }
@@ -686,8 +669,9 @@ public class HFileReaderV2 extends Abstr
         // It is important that we compute and pass onDiskSize to the block
         // reader so that it does not have to read the header separately to
         // figure out the size.
-        seekToBlock = reader2.fsBlockReader.readBlockData(previousBlockOffset,
-            seekToBlock.getOffset() - previousBlockOffset, -1, pread);
+        seekToBlock = reader.readBlock(previousBlockOffset,
+            seekToBlock.getOffset() - previousBlockOffset, cacheBlocks,
+            pread, isCompaction);
 
         // TODO shortcut: seek forward in this block to the last key of the
         // block.

Modified: hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java?rev=1178256&r1=1178255&r2=1178256&view=diff
==============================================================================
--- hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
(original)
+++ hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
Sun Oct  2 21:18:52 2011
@@ -163,8 +163,13 @@ public class TestCacheOnWrite {
         new EnumMap<BlockType, Integer>(BlockType.class);
 
     while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
-      HFileBlock block = reader.readBlockData(offset, prevBlock == null ? -1
-          : prevBlock.getNextBlockOnDiskSizeWithHeader(), -1, false);
+      long onDiskSize = -1;
+      if (prevBlock != null) {
+         onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader();
+      }
+      // Flags: cache the block, use pread, this is not a compaction.
+      HFileBlock block = reader.readBlock(offset, onDiskSize, true, true,
+          false);
       String blockCacheKey = HFile.getBlockCacheKey(reader.getName(), offset);
       boolean isCached = blockCache.getBlock(blockCacheKey, true) != null;
       boolean shouldBeCached = cowType.shouldBeCached(block.getBlockType());

Modified: hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java?rev=1178256&r1=1178255&r2=1178256&view=diff
==============================================================================
--- hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
(original)
+++ hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java
Sun Oct  2 21:18:52 2011
@@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.HBaseTest
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexReader;
 import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexChunk;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ClassSize;
@@ -121,37 +122,36 @@ public class TestHFileBlockIndex {
    * A wrapper around a block reader which only caches the results of the last
    * operation. Not thread-safe.
    */
-  private static class BlockReaderWrapper implements HFileBlock.BasicReader {
+  private static class BlockReaderWrapper implements HFile.CachingBlockReader {
 
-    private HFileBlock.BasicReader realReader;
+    private HFileBlock.FSReader realReader;
     private long prevOffset;
     private long prevOnDiskSize;
-    private long prevUncompressedSize;
     private boolean prevPread;
     private HFileBlock prevBlock;
 
     public int hitCount = 0;
     public int missCount = 0;
 
-    public BlockReaderWrapper(HFileBlock.BasicReader realReader) {
+    public BlockReaderWrapper(HFileBlock.FSReader realReader) {
       this.realReader = realReader;
     }
 
     @Override
-    public HFileBlock readBlockData(long offset, long onDiskSize,
-        int uncompressedSize, boolean pread) throws IOException {
+    public HFileBlock readBlock(long offset, long onDiskSize,
+        boolean cacheBlock, boolean pread, boolean isCompaction)
+        throws IOException {
       if (offset == prevOffset && onDiskSize == prevOnDiskSize &&
-          uncompressedSize == prevUncompressedSize && pread == prevPread) {
+          pread == prevPread) {
         hitCount += 1;
         return prevBlock;
       }
 
       missCount += 1;
       prevBlock = realReader.readBlockData(offset, onDiskSize,
-          uncompressedSize, pread);
+          -1, pread);
       prevOffset = offset;
       prevOnDiskSize = onDiskSize;
-      prevUncompressedSize = uncompressedSize;
       prevPread = pread;
 
       return prevBlock;
@@ -182,7 +182,8 @@ public class TestHFileBlockIndex {
     for (byte[] key : keys) {
       assertTrue(key != null);
       assertTrue(indexReader != null);
-      HFileBlock b = indexReader.seekToDataBlock(key, 0, key.length, null);
+      HFileBlock b = indexReader.seekToDataBlock(key, 0, key.length, null,
+          true, true, false);
       if (Bytes.BYTES_RAWCOMPARATOR.compare(key, firstKeyInFile) < 0) {
         assertTrue(b == null);
         ++i;



Mime
View raw message