fluo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject [incubator-fluo] branch master updated: fixes #830 (#875) optimized Bytes concatenation
Date Fri, 23 Jun 2017 15:23:06 GMT
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-fluo.git


The following commit(s) were added to refs/heads/master by this push:
     new d369af2  fixes #830 (#875) optimized Bytes concatenation
d369af2 is described below

commit d369af26c1fb05ebde50029ee66b8f0e37a3d0c1
Author: Christopher McTague <cjmctague@apache.org>
AuthorDate: Fri Jun 23 11:23:03 2017 -0400

    fixes #830 (#875) optimized Bytes concatenation
---
 .../apache/fluo/accumulo/util/ByteArrayUtil.java   | 101 ++++++++++++++++++---
 .../org/apache/fluo/core/util/ByteUtilTest.java    |   8 +-
 2 files changed, 93 insertions(+), 16 deletions(-)

diff --git a/modules/accumulo/src/main/java/org/apache/fluo/accumulo/util/ByteArrayUtil.java
b/modules/accumulo/src/main/java/org/apache/fluo/accumulo/util/ByteArrayUtil.java
index 33dc796..c1f4e5b 100644
--- a/modules/accumulo/src/main/java/org/apache/fluo/accumulo/util/ByteArrayUtil.java
+++ b/modules/accumulo/src/main/java/org/apache/fluo/accumulo/util/ByteArrayUtil.java
@@ -16,9 +16,7 @@
 package org.apache.fluo.accumulo.util;
 
 import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
-import java.io.DataOutputStream;
 import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStream;
@@ -98,21 +96,94 @@ public class ByteArrayUtil {
    * @param listOfBytes Bytes objects to concatenate
    * @return Bytes
    */
-  public static final byte[] concat(Bytes... listOfBytes) {
-    try {
-      // TODO calculate exact array size needed
-      ByteArrayOutputStream baos = new ByteArrayOutputStream();
-      DataOutputStream dos = new DataOutputStream(baos);
-
-      for (Bytes b : listOfBytes) {
-        WritableUtils.writeVInt(dos, b.length());
-        b.writeTo(dos);
+  public static byte[] concat(Bytes... listOfBytes) {
+    int offset = 0;
+    int size = 0;
+
+    for (Bytes b : listOfBytes) {
+      size += b.length() + checkVlen(b.length());
+    }
+
+    byte[] data = new byte[size];
+    for (Bytes b : listOfBytes) {
+      offset = writeVint(data, offset, b.length());
+      b.copyTo(0, b.length(), data, offset);
+      offset += b.length();
+    }
+    return data;
+  }
+
+  /**
+   * Writes a vInt directly to a byte array
+   * 
+   * @param dest The destination array for the vInt to be written to
+   * @param offset The location where to write the vInt to
+   * @param i The Value being written into byte array
+   * @return Returns the new offset location
+   */
+  public static int writeVint(byte[] dest, int offset, int i) {
+    if (i >= -112 && i <= 127) {
+      dest[offset++] = (byte) i;
+    } else {
+      int len = -112;
+      if (i < 0) {
+        i ^= -1L; // take one's complement'
+        len = -120;
       }
 
-      dos.close();
-      return baos.toByteArray();
-    } catch (IOException e) {
-      throw new RuntimeException(e);
+      long tmp = i;
+      while (tmp != 0) {
+        tmp = tmp >> 8;
+        len--;
+      }
+
+      dest[offset++] = (byte) len;
+
+      len = (len < -120) ? -(len + 120) : -(len + 112);
+
+      for (int idx = len; idx != 0; idx--) {
+        int shiftbits = (idx - 1) * 8;
+        long mask = 0xFFL << shiftbits;
+        dest[offset++] = (byte) ((i & mask) >> shiftbits);
+      }
+    }
+
+    return offset;
+  }
+
+  /**
+   * Determines the number bytes required to store a variable length
+   * 
+   * @param i length of Bytes
+   * @return number of bytes needed
+   */
+  public static int checkVlen(int i) {
+    int count = 0;
+    if (i >= -112 && i <= 127) {
+      return 1;
+    } else {
+      int len = -112;
+      if (i < 0) {
+        i ^= -1L; // take one's complement'
+        len = -120;
+      }
+
+      long tmp = i;
+      while (tmp != 0) {
+        tmp = tmp >> 8;
+        len--;
+      }
+
+      count++;
+
+      len = (len < -120) ? -(len + 120) : -(len + 112);
+
+      while (len != 0) {
+        count++;
+        len--;
+      }
+
+      return count;
     }
   }
 
diff --git a/modules/core/src/test/java/org/apache/fluo/core/util/ByteUtilTest.java b/modules/core/src/test/java/org/apache/fluo/core/util/ByteUtilTest.java
index be575ac..768c572 100644
--- a/modules/core/src/test/java/org/apache/fluo/core/util/ByteUtilTest.java
+++ b/modules/core/src/test/java/org/apache/fluo/core/util/ByteUtilTest.java
@@ -54,12 +54,18 @@ public class ByteUtilTest {
     Bytes b1 = Bytes.of("str1");
     Bytes b2 = Bytes.of("string2");
     Bytes b3 = Bytes.of("s3");
-    byte[] ball = ByteArrayUtil.concat(b1, b2, b3);
+    Bytes b4 =
+        Bytes.of("testinggreaterthan128characterstestinggreaterthan128characters"
+            + "testinggreaterthan128characterstestinggreaterthan128characters"
+            + "testinggreaterthan128characters"); // 155 length
+
+    byte[] ball = ByteArrayUtil.concat(b1, b2, b3, b4);
 
     List<Bytes> blist = ByteArrayUtil.split(ball);
 
     Assert.assertEquals(b1, blist.get(0));
     Assert.assertEquals(b2, blist.get(1));
     Assert.assertEquals(b3, blist.get(2));
+    Assert.assertEquals(b4, blist.get(3));
   }
 }

-- 
To stop receiving notification emails like this one, please contact
['"commits@fluo.apache.org" <commits@fluo.apache.org>'].

Mime
View raw message