crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject crunch git commit: CRUNCH-614: Fix HFileUtils.writeToHFilesForIncrementalLoad slowed dramatically by copying KeyValue byte array. Contributed by Ben Roling.
Date Sat, 30 Jul 2016 00:47:22 GMT
Repository: crunch
Updated Branches:
  refs/heads/master 8ca28db25 -> b0491f20a


CRUNCH-614: Fix HFileUtils.writeToHFilesForIncrementalLoad slowed dramatically by copying
KeyValue byte array. Contributed by Ben Roling.


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/b0491f20
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/b0491f20
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/b0491f20

Branch: refs/heads/master
Commit: b0491f20ac9f671020fcbc3f3118cd4eda013c57
Parents: 8ca28db
Author: Josh Wills <jwills@apache.org>
Authored: Fri Jul 29 17:47:09 2016 -0700
Committer: Josh Wills <jwills@apache.org>
Committed: Fri Jul 29 17:47:09 2016 -0700

----------------------------------------------------------------------
 .../java/org/apache/crunch/io/hbase/HFileUtils.java  | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/b0491f20/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileUtils.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileUtils.java b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileUtils.java
index 57fdffb..4cbe1c5 100644
--- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileUtils.java
+++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileUtils.java
@@ -260,16 +260,15 @@ public final class HFileUtils {
 
     @Override
     public int compare(byte[] left, int loffset, int llength, byte[] right, int roffset,
int rlength) {
-      // BytesWritable serialize length in first 4 bytes.
-      // We simply ignore it here, because KeyValue has its own size serialized.
-      if (llength < 4) {
+      // BytesWritable and KeyValue each serialize 4 bytes to indicate length
+      if (llength < 8) {
         throw new AssertionError("Too small llength: " + llength);
       }
-      if (rlength < 4) {
+      if (rlength < 8) {
         throw new AssertionError("Too small rlength: " + rlength);
       }
-      Cell leftKey = HBaseTypes.bytesToKeyValue(left, loffset + 4, llength - 4);
-      Cell rightKey = HBaseTypes.bytesToKeyValue(right, roffset + 4, rlength - 4);
+      Cell leftKey = new KeyValue(left, loffset + 8, llength - 8);
+      Cell rightKey = new KeyValue(right, roffset + 8, rlength - 8);
 
       byte[] lRow = leftKey.getRow();
       byte[] rRow = rightKey.getRow();
@@ -284,8 +283,8 @@ public final class HFileUtils {
     @Override
     public int compare(BytesWritable left, BytesWritable right) {
       return KeyValue.COMPARATOR.compare(
-          HBaseTypes.bytesToKeyValue(left),
-          HBaseTypes.bytesToKeyValue(right));
+          new KeyValue(left.getBytes(), 4, left.getLength() - 4),
+	  new KeyValue(right.getBytes(), 4, right.getLength() - 4));
     }
   }
 


Mime
View raw message