hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ramkris...@apache.org
Subject hbase git commit: HBASE-15204 Try to estimate the cell count for adding into WALEdit (Ram)
Date Fri, 05 Feb 2016 08:54:58 GMT
Repository: hbase
Updated Branches:
  refs/heads/master bb71446e1 -> 6f6a8ed71


HBASE-15204 Try to estimate the cell count for adding into WALEdit (Ram)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6f6a8ed7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6f6a8ed7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6f6a8ed7

Branch: refs/heads/master
Commit: 6f6a8ed71fe98b83e8a8db974fc15b0d8597b174
Parents: bb71446
Author: ramkrishna <ramkrishna.s.vasudevan@gmail.com>
Authored: Fri Feb 5 14:23:36 2016 +0530
Committer: ramkrishna <ramkrishna.s.vasudevan@gmail.com>
Committed: Fri Feb 5 14:24:38 2016 +0530

----------------------------------------------------------------------
 .../hadoop/hbase/regionserver/HRegion.java      | 25 +++++++++++++-------
 .../hadoop/hbase/regionserver/wal/WALEdit.java  |  8 ++++++-
 2 files changed, 23 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/6f6a8ed7/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index f03c205..86f4a1b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -2906,25 +2906,26 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver,
Regi
    *         OperationStatusCode and the exceptionMessage if any.
    * @throws IOException
    */
-  OperationStatus[] batchMutate(BatchOperationInProgress<?> batchOp) throws IOException
{
+  OperationStatus[] batchMutate(BatchOperationInProgress<?> batchOp)
+      throws IOException {
     boolean initialized = false;
     Operation op = batchOp.isInReplay() ? Operation.REPLAY_BATCH_MUTATE : Operation.BATCH_MUTATE;
     startRegionOperation(op);
+    int cellCountFromCP = 0;
     try {
       while (!batchOp.isDone()) {
         if (!batchOp.isInReplay()) {
           checkReadOnly();
         }
         checkResources();
-
         if (!initialized) {
           this.writeRequestsCount.add(batchOp.operations.length);
           if (!batchOp.isInReplay()) {
-            doPreMutationHook(batchOp);
+            cellCountFromCP = doPreMutationHook(batchOp);
           }
           initialized = true;
         }
-        long addedSize = doMiniBatchMutation(batchOp);
+        long addedSize = doMiniBatchMutation(batchOp, cellCountFromCP);
         long newSize = this.addAndGetGlobalMemstoreSize(addedSize);
         if (isFlushSize(newSize)) {
           requestFlush();
@@ -2937,10 +2938,11 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver,
Regi
   }
 
 
-  private void doPreMutationHook(BatchOperationInProgress<?> batchOp)
+  private int doPreMutationHook(BatchOperationInProgress<?> batchOp)
       throws IOException {
     /* Run coprocessor pre hook outside of locks to avoid deadlock */
     WALEdit walEdit = new WALEdit();
+    int cellCount = 0;
     if (coprocessorHost != null) {
       for (int i = 0 ; i < batchOp.operations.length; i++) {
         Mutation m = batchOp.getMutation(i);
@@ -2970,14 +2972,17 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver,
Regi
         }
         if (!walEdit.isEmpty()) {
           batchOp.walEditsFromCoprocessors[i] = walEdit;
+          cellCount += walEdit.size();
           walEdit = new WALEdit();
         }
       }
     }
+    return cellCount;
   }
 
   @SuppressWarnings("unchecked")
-  private long doMiniBatchMutation(BatchOperationInProgress<?> batchOp) throws IOException
{
+  private long doMiniBatchMutation(BatchOperationInProgress<?> batchOp, int cellCount)
+      throws IOException {
     boolean isInReplay = batchOp.isInReplay();
     // variable to note if all Put items are for the same CF -- metrics related
     boolean putsCfSetConsistent = true;
@@ -2989,7 +2994,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver,
Regi
     Set<byte[]> deletesCfSet = null;
 
     long currentNonceGroup = HConstants.NO_NONCE, currentNonce = HConstants.NO_NONCE;
-    WALEdit walEdit = new WALEdit(isInReplay);
+    WALEdit walEdit = null;
     MultiVersionConcurrencyControl.WriteEntry writeEntry = null;
     long txid = 0;
     boolean doRollBackMemstore = false;
@@ -3020,7 +3025,6 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver,
Regi
         Map<byte[], List<Cell>> familyMap = mutation.getFamilyCellMap();
         // store the family map reference to allow for mutations
         familyMaps[lastIndexExclusive] = familyMap;
-
         // skip anything that "ran" already
         if (batchOp.retCodeDetails[lastIndexExclusive].getOperationStatusCode()
             != OperationStatusCode.NOT_RUN) {
@@ -3127,8 +3131,11 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver,
Regi
           noOfDeletes++;
         }
         rewriteCellTags(familyMaps[i], mutation);
+        for (List<Cell> cells : familyMaps[i].values()) {
+          cellCount += cells.size();
+        }
       }
-
+      walEdit = new WALEdit(cellCount);
       lock(this.updatesLock.readLock(), numReadyToWrite);
       locked = true;
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/6f6a8ed7/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
index cea2ee7..1a87447 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
@@ -99,7 +99,7 @@ public class WALEdit implements Writable, HeapSize {
   private final int VERSION_2 = -1;
   private final boolean isReplay;
 
-  private ArrayList<Cell> cells = new ArrayList<Cell>(1);
+  private ArrayList<Cell> cells = null;
 
   public static final WALEdit EMPTY_WALEDIT = new WALEdit();
 
@@ -118,6 +118,12 @@ public class WALEdit implements Writable, HeapSize {
 
   public WALEdit(boolean isReplay) {
     this.isReplay = isReplay;
+    cells = new ArrayList<Cell>(1);
+  }
+
+  public WALEdit(int cellCount) {
+    this.isReplay = false;
+    cells = new ArrayList<Cell>(cellCount);
   }
 
   /**


Mime
View raw message