Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@hbase.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: anoopsamjohn@apache.org
To: commits@hbase.apache.org
Message-Id: <ff85af1edf054b4d85b05ee49d0e7ed1@git.apache.org>
Subject: hbase git commit: HBASE-17757 Unify blocksize after encoding to
 decrease memory fragment
Date: Tue,  2 May 2017 04:39:21 +0000 (UTC)
archived-at: Tue, 02 May 2017 04:39:24 -0000

Repository: hbase
Updated Branches:
  refs/heads/branch-1 961bb7325 -> 262c66f61


HBASE-17757 Unify blocksize after encoding to decrease memory fragment

Signed-off-by: anoopsamjohn <anoopsamjohn@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/262c66f6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/262c66f6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/262c66f6

Branch: refs/heads/branch-1
Commit: 262c66f614d436c15b47ed4ae2ba33ed913c3cac
Parents: 961bb73
Author: Allan Yang <allanwin@163.com>
Authored: Sun Apr 30 12:01:26 2017 +0800
Committer: anoopsamjohn <anoopsamjohn@gmail.com>
Committed: Tue May 2 10:06:33 2017 +0530

----------------------------------------------------------------------
 .../hadoop/hbase/io/hfile/HFileBlock.java       | 20 ++++++++++++++++
 .../hadoop/hbase/io/hfile/HFileWriterV2.java    | 24 +++++++++++++++-----
 2 files changed, 38 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/262c66f6/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
index b8629da..3b014b9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
@@ -844,6 +844,10 @@ public class HFileBlock implements Cacheable {
     // includes the header size also.
     private int unencodedDataSizeWritten;
 
+    // Size of actual data being written. considering the block encoding. This
+    // includes the header size also.
+    private int encodedDataSizeWritten;
+
     /**
      * Bytes to be written to the file system, including the header. Compressed
      * if compression is turned on. It also includes the checksum data that
@@ -931,6 +935,7 @@ public class HFileBlock implements Cacheable {
         this.dataBlockEncoder.startBlockEncoding(dataBlockEncodingCtx, userDataStream);
       }
       this.unencodedDataSizeWritten = 0;
+      this.encodedDataSizeWritten = 0;
       return userDataStream;
     }
 
@@ -941,8 +946,10 @@ public class HFileBlock implements Cacheable {
      */
     void write(Cell cell) throws IOException{
       expectState(State.WRITING);
+      int posBeforeEncode = this.userDataStream.size();
       this.unencodedDataSizeWritten +=
           this.dataBlockEncoder.encode(cell, dataBlockEncodingCtx, this.userDataStream);
+      this.encodedDataSizeWritten += this.userDataStream.size() - posBeforeEncode;
     }
 
     /**
@@ -1179,6 +1186,19 @@ public class HFileBlock implements Cacheable {
      *
      * @return the number of bytes written
      */
+    public int encodedBlockSizeWritten() {
+      if (state != State.WRITING)
+        return 0;
+      return this.encodedDataSizeWritten;
+    }
+
+    /**
+     * Returns the number of bytes written into the current block so far, or
+     * zero if not writing the block at the moment. Note that this will return
+     * zero in the "block ready" state as well.
+     *
+     * @return the number of bytes written
+     */
     int blockSizeWritten() {
       if (state != State.WRITING) return 0;
       return this.unencodedDataSizeWritten;

http://git-wip-us.apache.org/repos/asf/hbase/blob/262c66f6/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
index 9e68dc3..394cd44 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
@@ -93,6 +93,14 @@ public class HFileWriterV2 extends AbstractHFileWriter {
   /** warn on cell with tags */
   private static boolean warnCellWithTags = true;
 
+
+  /** if this feature is enabled, preCalculate encoded data size before real encoding happens*/
+  public static final String UNIFIED_ENCODED_BLOCKSIZE_RATIO = "hbase.writer.unified.encoded.blocksize.ratio";
+
+  /** Block size limit after encoding, used to unify encoded block Cache entry size*/
+  private final int encodedBlockSizeLimit;
+
+
   static class WriterFactoryV2 extends HFile.WriterFactory {
     WriterFactoryV2(Configuration conf, CacheConfig cacheConf) {
       super(conf, cacheConf);
@@ -115,6 +123,8 @@ public class HFileWriterV2 extends AbstractHFileWriter {
     super(cacheConf,
         ostream == null ? createOutputStream(conf, fs, path, null) : ostream,
         path, comparator, context);
+    float encodeBlockSizeRatio = conf.getFloat(UNIFIED_ENCODED_BLOCKSIZE_RATIO, 1f);
+    this.encodedBlockSizeLimit = (int)(hFileContext.getBlocksize() * encodeBlockSizeRatio);
     finishInit(conf);
   }
 
@@ -147,12 +157,14 @@ public class HFileWriterV2 extends AbstractHFileWriter {
    * @throws IOException
    */
   protected void checkBlockBoundary() throws IOException {
-    if (fsBlockWriter.blockSizeWritten() < hFileContext.getBlocksize())
-      return;
-
-    finishBlock();
-    writeInlineBlocks(false);
-    newBlock();
+    //for encoder like prefixTree, encoded size is not available, so we have to compare both encoded size
+    //and unencoded size to blocksize limit.
+    if (fsBlockWriter.encodedBlockSizeWritten() >= encodedBlockSizeLimit
+        || fsBlockWriter.blockSizeWritten() >= hFileContext.getBlocksize()) {
+      finishBlock();
+      writeInlineBlocks(false);
+      newBlock();
+    }
   }
 
   /** Clean up the current data block */