hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject svn commit: r1153634 [1/4] - in /hbase/trunk: ./ src/main/java/org/apache/hadoop/hbase/ src/main/java/org/apache/hadoop/hbase/io/hfile/ src/main/java/org/apache/hadoop/hbase/mapreduce/ src/main/java/org/apache/hadoop/hbase/regionserver/ src/main/java/o...
Date Wed, 03 Aug 2011 19:59:54 GMT
Author: tedyu
Date: Wed Aug  3 19:59:48 2011
New Revision: 1153634

URL: http://svn.apache.org/viewvc?rev=1153634&view=rev
Log:
HBASE-3857  Change the HFile Format (Mikhail & Liyin)

Modified:
    hbase/trunk/CHANGES.txt
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/BloomFilter.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/Hash.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/RecoverableZooKeeper.java
    hbase/trunk/src/main/resources/hbase-default.xml
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/RandomSeek.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestByteBloomFilter.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java

Modified: hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1153634&r1=1153633&r2=1153634&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Wed Aug  3 19:59:48 2011
@@ -395,7 +395,7 @@ Release 0.91.0 - Unreleased
    HBASE-3691  Add compressor support for 'snappy', google's compressor
                (Nichole Treadway and Nicholas Telford)
    HBASE-2233  Support both Hadoop 0.20 and 0.22
-
+   HBASE-3857  Change the HFile Format (Mikhail & Liyin)
 
 Release 0.90.4 - Unreleased
 

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java?rev=1153634&r1=1153633&r2=1153634&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java Wed Aug  3 19:59:48
2011
@@ -83,13 +83,29 @@ implements WritableComparable<HServerLoa
     private int storefileSizeMB;
     /** the current size of the memstore for the region, in MB */
     private int memstoreSizeMB;
-    /** the current total size of storefile indexes for the region, in MB */
+
+    /**
+     * The current total size of root-level store file indexes for the region,
+     * in MB. The same as {@link #rootIndexSizeKB} but in MB.
+     */
     private int storefileIndexSizeMB;
     /** the current total read requests made to region */
     private int readRequestsCount;
     /** the current total write requests made to region */
     private int writeRequestsCount;
 
+    /** The current total size of root-level indexes for the region, in KB. */
+    private int rootIndexSizeKB;
+
+    /** The total size of all index blocks, not just the root level, in KB. */
+    private int totalStaticIndexSizeKB;
+
+    /**
+     * The total size of all Bloom filter blocks, not just loaded into the
+     * block cache, in KB.
+     */
+    private int totalStaticBloomSizeKB;
+
     /**
      * Constructor, for Writable
      */
@@ -111,6 +127,8 @@ implements WritableComparable<HServerLoa
         final int storefiles, final int storeUncompressedSizeMB,
         final int storefileSizeMB,
         final int memstoreSizeMB, final int storefileIndexSizeMB,
+        final int rootIndexSizeKB, final int totalStaticIndexSizeKB,
+        final int totalStaticBloomSizeKB,
         final int readRequestsCount, final int writeRequestsCount) {
       this.name = name;
       this.stores = stores;
@@ -119,6 +137,9 @@ implements WritableComparable<HServerLoa
       this.storefileSizeMB = storefileSizeMB;
       this.memstoreSizeMB = memstoreSizeMB;
       this.storefileIndexSizeMB = storefileIndexSizeMB;
+      this.rootIndexSizeKB = rootIndexSizeKB;
+      this.totalStaticIndexSizeKB = totalStaticIndexSizeKB;
+      this.totalStaticBloomSizeKB = totalStaticBloomSizeKB;
       this.readRequestsCount = readRequestsCount;
       this.writeRequestsCount = writeRequestsCount;
     }
@@ -263,6 +284,9 @@ implements WritableComparable<HServerLoa
       this.storefileIndexSizeMB = in.readInt();
       this.readRequestsCount = in.readInt();
       this.writeRequestsCount = in.readInt();
+      this.rootIndexSizeKB = in.readInt();
+      this.totalStaticIndexSizeKB = in.readInt();
+      this.totalStaticBloomSizeKB = in.readInt();
     }
 
     public void write(DataOutput out) throws IOException {
@@ -278,6 +302,9 @@ implements WritableComparable<HServerLoa
       out.writeInt(storefileIndexSizeMB);
       out.writeInt(readRequestsCount);
       out.writeInt(writeRequestsCount);
+      out.writeInt(rootIndexSizeKB);
+      out.writeInt(totalStaticIndexSizeKB);
+      out.writeInt(totalStaticBloomSizeKB);
     }
 
     /**
@@ -306,6 +333,12 @@ implements WritableComparable<HServerLoa
           Long.valueOf(this.readRequestsCount));
       sb = Strings.appendKeyValue(sb, "writeRequestsCount",
           Long.valueOf(this.writeRequestsCount));
+      sb = Strings.appendKeyValue(sb, "rootIndexSizeKB",
+          Integer.valueOf(this.rootIndexSizeKB));
+      sb = Strings.appendKeyValue(sb, "totalStaticIndexSizeKB",
+          Integer.valueOf(this.totalStaticIndexSizeKB));
+      sb = Strings.appendKeyValue(sb, "totalStaticBloomSizeKB",
+        Integer.valueOf(this.totalStaticBloomSizeKB));
       return sb.toString();
     }
   }

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java?rev=1153634&r1=1153633&r2=1153634&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java Wed Aug  3
19:59:48 2011
@@ -19,12 +19,15 @@
  */
 package org.apache.hadoop.hbase.io.hfile;
 
-import java.nio.ByteBuffer;
-
+import org.apache.hadoop.hbase.io.HeapSize;
 import org.apache.hadoop.hbase.io.hfile.LruBlockCache.CacheStats;
 
 /**
- * Block cache interface.
+ * Block cache interface. Anything that implements the {@link HeapSize}
+ * interface can be put in the cache, because item size is all the cache
+ * cares about. We might move to a more specialized "cacheable" interface
+ * in the future.
+ *
  * TODO: Add filename or hash of filename to block cache key.
  */
 public interface BlockCache {
@@ -34,14 +37,14 @@ public interface BlockCache {
    * @param buf The block contents wrapped in a ByteBuffer.
    * @param inMemory Whether block should be treated as in-memory
    */
-  public void cacheBlock(String blockName, ByteBuffer buf, boolean inMemory);
+  public void cacheBlock(String blockName, HeapSize buf, boolean inMemory);
 
   /**
    * Add block to cache (defaults to not in-memory).
    * @param blockName Zero-based file block number.
    * @param buf The block contents wrapped in a ByteBuffer.
    */
-  public void cacheBlock(String blockName, ByteBuffer buf);
+  public void cacheBlock(String blockName, HeapSize buf);
 
   /**
    * Fetch block from cache.
@@ -49,7 +52,7 @@ public interface BlockCache {
    * @param caching Whether this request has caching enabled (used for stats)
    * @return Block or null if block is not in the cache.
    */
-  public ByteBuffer getBlock(String blockName, boolean caching);
+  public HeapSize getBlock(String blockName, boolean caching);
 
   /**
    * Evict block from cache.
@@ -59,6 +62,17 @@ public interface BlockCache {
   public boolean evictBlock(String blockName);
 
   /**
+   * Evicts all blocks with name starting with the given prefix. This is
+   * necessary in cases we need to evict all blocks that belong to a particular
+   * HFile. In HFile v2 all blocks consist of the storefile name (UUID), an
+   * underscore, and the block offset in the file. An efficient implementation
+   * would avoid scanning all blocks in the cache.
+   *
+   * @return the number of blocks evicted
+   */
+  public int evictBlocksByPrefix(String string);
+
+  /**
    * Get the statistics for this block cache.
    * @return
    */

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java?rev=1153634&r1=1153633&r2=1153634&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java Wed Aug  3
19:59:48 2011
@@ -55,22 +55,22 @@ public class CachedBlock implements Heap
   };
 
   private final String blockName;
-  private final ByteBuffer buf;
+  private final HeapSize buf;
   private volatile long accessTime;
   private long size;
   private BlockPriority priority;
 
-  public CachedBlock(String blockName, ByteBuffer buf, long accessTime) {
+  public CachedBlock(String blockName, HeapSize buf, long accessTime) {
     this(blockName, buf, accessTime, false);
   }
 
-  public CachedBlock(String blockName, ByteBuffer buf, long accessTime,
+  public CachedBlock(String blockName, HeapSize buf, long accessTime,
       boolean inMemory) {
     this.blockName = blockName;
     this.buf = buf;
     this.accessTime = accessTime;
-    this.size = ClassSize.align(blockName.length()) +
-    ClassSize.align(buf.capacity()) + PER_BLOCK_OVERHEAD;
+    this.size = ClassSize.align(blockName.length())
+        + ClassSize.align(buf.heapSize()) + PER_BLOCK_OVERHEAD;
     if(inMemory) {
       this.priority = BlockPriority.MEMORY;
     } else {
@@ -97,7 +97,7 @@ public class CachedBlock implements Heap
     return this.accessTime < that.accessTime ? 1 : -1;
   }
 
-  public ByteBuffer getBuffer() {
+  public HeapSize getBuffer() {
     return this.buf;
   }
 



Mime
View raw message