Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id DF1301878E for ; Wed, 27 Jan 2016 17:43:22 +0000 (UTC) Received: (qmail 10350 invoked by uid 500); 27 Jan 2016 17:43:13 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 10309 invoked by uid 500); 27 Jan 2016 17:43:13 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 10292 invoked by uid 99); 27 Jan 2016 17:43:13 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 27 Jan 2016 17:43:13 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 3AC9BDFF96; Wed, 27 Jan 2016 17:43:13 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: tedyu@apache.org To: commits@hbase.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hbase git commit: HBASE-15171 Avoid counting duplicate kv and generating lots of small hfiles in PutSortReducer (Yu Li) Date: Wed, 27 Jan 2016 17:43:13 +0000 (UTC) Repository: hbase Updated Branches: refs/heads/master 845d00a16 -> 47c414794 HBASE-15171 Avoid counting duplicate kv and generating lots of small hfiles in PutSortReducer (Yu Li) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/47c41479 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/47c41479 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/47c41479 Branch: refs/heads/master Commit: 47c41479401ea0aadfa3c3776fe2930bb8e9710d Parents: 845d00a Author: tedyu Authored: Wed Jan 27 09:42:56 2016 -0800 Committer: tedyu Committed: Wed Jan 27 09:42:56 2016 -0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hbase/mapreduce/PutSortReducer.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/47c41479/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java index 0a27056..d18ea34 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java @@ -22,12 +22,12 @@ import java.util.Iterator; import java.util.List; import java.util.TreeSet; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.mapreduce.Reducer; @@ -68,9 +68,11 @@ public class PutSortReducer extends for (Cell cell: cells) { KeyValue kv = KeyValueUtil.ensureKeyValue(cell); map.add(kv); - curSize += kv.heapSize(); } } + for(KeyValue kv: map){ + curSize +=kv.heapSize(); + } } context.setStatus("Read " + map.size() + " entries of " + map.getClass() + "(" + StringUtils.humanReadableInt(curSize) + ")");