Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 86EEE18275 for ; Fri, 5 Feb 2016 08:54:58 +0000 (UTC) Received: (qmail 15435 invoked by uid 500); 5 Feb 2016 08:54:58 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 15395 invoked by uid 500); 5 Feb 2016 08:54:58 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 15386 invoked by uid 99); 5 Feb 2016 08:54:58 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 05 Feb 2016 08:54:58 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 473FDDFF96; Fri, 5 Feb 2016 08:54:58 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ramkrishna@apache.org To: commits@hbase.apache.org Message-Id: <03a9c1de19554c11b87135c6ed69daaf@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hbase git commit: HBASE-15204 Try to estimate the cell count for adding into WALEdit (Ram) Date: Fri, 5 Feb 2016 08:54:58 +0000 (UTC) Repository: hbase Updated Branches: refs/heads/master bb71446e1 -> 6f6a8ed71 HBASE-15204 Try to estimate the cell count for adding into WALEdit (Ram) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6f6a8ed7 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6f6a8ed7 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6f6a8ed7 Branch: refs/heads/master Commit: 6f6a8ed71fe98b83e8a8db974fc15b0d8597b174 Parents: bb71446 Author: ramkrishna Authored: Fri Feb 5 14:23:36 2016 +0530 Committer: ramkrishna Committed: Fri Feb 5 14:24:38 2016 +0530 ---------------------------------------------------------------------- .../hadoop/hbase/regionserver/HRegion.java | 25 +++++++++++++------- .../hadoop/hbase/regionserver/wal/WALEdit.java | 8 ++++++- 2 files changed, 23 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/6f6a8ed7/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index f03c205..86f4a1b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -2906,25 +2906,26 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi * OperationStatusCode and the exceptionMessage if any. * @throws IOException */ - OperationStatus[] batchMutate(BatchOperationInProgress batchOp) throws IOException { + OperationStatus[] batchMutate(BatchOperationInProgress batchOp) + throws IOException { boolean initialized = false; Operation op = batchOp.isInReplay() ? Operation.REPLAY_BATCH_MUTATE : Operation.BATCH_MUTATE; startRegionOperation(op); + int cellCountFromCP = 0; try { while (!batchOp.isDone()) { if (!batchOp.isInReplay()) { checkReadOnly(); } checkResources(); - if (!initialized) { this.writeRequestsCount.add(batchOp.operations.length); if (!batchOp.isInReplay()) { - doPreMutationHook(batchOp); + cellCountFromCP = doPreMutationHook(batchOp); } initialized = true; } - long addedSize = doMiniBatchMutation(batchOp); + long addedSize = doMiniBatchMutation(batchOp, cellCountFromCP); long newSize = this.addAndGetGlobalMemstoreSize(addedSize); if (isFlushSize(newSize)) { requestFlush(); @@ -2937,10 +2938,11 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi } - private void doPreMutationHook(BatchOperationInProgress batchOp) + private int doPreMutationHook(BatchOperationInProgress batchOp) throws IOException { /* Run coprocessor pre hook outside of locks to avoid deadlock */ WALEdit walEdit = new WALEdit(); + int cellCount = 0; if (coprocessorHost != null) { for (int i = 0 ; i < batchOp.operations.length; i++) { Mutation m = batchOp.getMutation(i); @@ -2970,14 +2972,17 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi } if (!walEdit.isEmpty()) { batchOp.walEditsFromCoprocessors[i] = walEdit; + cellCount += walEdit.size(); walEdit = new WALEdit(); } } } + return cellCount; } @SuppressWarnings("unchecked") - private long doMiniBatchMutation(BatchOperationInProgress batchOp) throws IOException { + private long doMiniBatchMutation(BatchOperationInProgress batchOp, int cellCount) + throws IOException { boolean isInReplay = batchOp.isInReplay(); // variable to note if all Put items are for the same CF -- metrics related boolean putsCfSetConsistent = true; @@ -2989,7 +2994,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi Set deletesCfSet = null; long currentNonceGroup = HConstants.NO_NONCE, currentNonce = HConstants.NO_NONCE; - WALEdit walEdit = new WALEdit(isInReplay); + WALEdit walEdit = null; MultiVersionConcurrencyControl.WriteEntry writeEntry = null; long txid = 0; boolean doRollBackMemstore = false; @@ -3020,7 +3025,6 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi Map> familyMap = mutation.getFamilyCellMap(); // store the family map reference to allow for mutations familyMaps[lastIndexExclusive] = familyMap; - // skip anything that "ran" already if (batchOp.retCodeDetails[lastIndexExclusive].getOperationStatusCode() != OperationStatusCode.NOT_RUN) { @@ -3127,8 +3131,11 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi noOfDeletes++; } rewriteCellTags(familyMaps[i], mutation); + for (List cells : familyMaps[i].values()) { + cellCount += cells.size(); + } } - + walEdit = new WALEdit(cellCount); lock(this.updatesLock.readLock(), numReadyToWrite); locked = true; http://git-wip-us.apache.org/repos/asf/hbase/blob/6f6a8ed7/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java index cea2ee7..1a87447 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java @@ -99,7 +99,7 @@ public class WALEdit implements Writable, HeapSize { private final int VERSION_2 = -1; private final boolean isReplay; - private ArrayList cells = new ArrayList(1); + private ArrayList cells = null; public static final WALEdit EMPTY_WALEDIT = new WALEdit(); @@ -118,6 +118,12 @@ public class WALEdit implements Writable, HeapSize { public WALEdit(boolean isReplay) { this.isReplay = isReplay; + cells = new ArrayList(1); + } + + public WALEdit(int cellCount) { + this.isReplay = false; + cells = new ArrayList(cellCount); } /**