hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-15171 Avoid counting duplicate kv and generating lots of small hfiles in PutSortReducer (Yu Li)
Date Wed, 27 Jan 2016 17:43:13 GMT
Repository: hbase
Updated Branches:
  refs/heads/master 845d00a16 -> 47c414794


HBASE-15171 Avoid counting duplicate kv and generating lots of small hfiles in PutSortReducer
(Yu Li)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/47c41479
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/47c41479
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/47c41479

Branch: refs/heads/master
Commit: 47c41479401ea0aadfa3c3776fe2930bb8e9710d
Parents: 845d00a
Author: tedyu <yuzhihong@gmail.com>
Authored: Wed Jan 27 09:42:56 2016 -0800
Committer: tedyu <yuzhihong@gmail.com>
Committed: Wed Jan 27 09:42:56 2016 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hbase/mapreduce/PutSortReducer.java    | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/47c41479/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
index 0a27056..d18ea34 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
@@ -22,12 +22,12 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.TreeSet;
 
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellComparator;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.mapreduce.Reducer;
@@ -68,9 +68,11 @@ public class PutSortReducer extends
           for (Cell cell: cells) {
             KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
             map.add(kv);
-            curSize += kv.heapSize();
           }
         }
+        for(KeyValue kv: map){
+          curSize +=kv.heapSize();
+        }
       }
       context.setStatus("Read " + map.size() + " entries of " + map.getClass()
           + "(" + StringUtils.humanReadableInt(curSize) + ")");


Mime
View raw message