aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kevi...@apache.org
Subject git commit: Use BufferedOutputStream for deflater
Date Thu, 25 Sep 2014 22:23:31 GMT
Repository: incubator-aurora
Updated Branches:
  refs/heads/master 38e4be2e7 -> 8edc34b11


Use BufferedOutputStream for deflater

Testing Done:
./gradlew build

Microbenchmark with a scheduler backup from my production environment.

Before:
# Deflater.DEFAULT_COMPRESSION
# (No BufferedOutputStream)
# DeflaterOutputStream(512B)
deflate: 16664ms
total:17219ms
compression ratio: 647270930/92690409 = 698.31%

After:
# Deflater.DEFAULT_COMPRESSION
# BufferedOutputStream(256KiB)
# DeflaterOutputStream(256KiB)
deflate: 11807ms (4857ms speedup)
total: 12379ms (4840ms speedup)
compression ratio: 647270930/92690409 = 698.31% (0% improvement)

Bugs closed: AURORA-722

Reviewed at https://reviews.apache.org/r/26012/


Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/8edc34b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/8edc34b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/8edc34b1

Branch: refs/heads/master
Commit: 8edc34b1196543206d79c915b7607b19f5c63b70
Parents: 38e4be2
Author: Kevin Sweeney <kevints@apache.org>
Authored: Thu Sep 25 15:22:49 2014 -0700
Committer: Kevin Sweeney <kevints@apache.org>
Committed: Thu Sep 25 15:22:49 2014 -0700

----------------------------------------------------------------------
 .../apache/aurora/codec/ThriftBinaryCodec.java  | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/8edc34b1/src/main/java/org/apache/aurora/codec/ThriftBinaryCodec.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/codec/ThriftBinaryCodec.java b/src/main/java/org/apache/aurora/codec/ThriftBinaryCodec.java
index 45cf7ec..65e986e 100644
--- a/src/main/java/org/apache/aurora/codec/ThriftBinaryCodec.java
+++ b/src/main/java/org/apache/aurora/codec/ThriftBinaryCodec.java
@@ -13,14 +13,19 @@
  */
 package org.apache.aurora.codec;
 
+import java.io.BufferedOutputStream;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.lang.reflect.InvocationTargetException;
+import java.util.zip.Deflater;
 import java.util.zip.DeflaterOutputStream;
 import java.util.zip.InflaterInputStream;
 
 import javax.annotation.Nullable;
 
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Data;
+
 import org.apache.thrift.TBase;
 import org.apache.thrift.TDeserializer;
 import org.apache.thrift.TException;
@@ -122,6 +127,10 @@ public final class ThriftBinaryCodec {
     }
   }
 
+  // See http://www.zlib.net/zlib_how.html
+  // "If the memory is available, buffers sizes on the order of 128K or 256K bytes should
be used."
+  private static final int DEFLATER_BUFFER_SIZE = Amount.of(256, Data.KB).as(Data.BYTES);
+
   /**
    * Encodes a thrift object into a DEFLATE-compressed binary array.
    *
@@ -134,7 +143,16 @@ public final class ThriftBinaryCodec {
 
     ByteArrayOutputStream outBytes = new ByteArrayOutputStream();
     try {
-      TTransport transport = new TIOStreamTransport(new DeflaterOutputStream(outBytes));
+      // NOTE: Buffering is needed here for performance.
+      // There are actually 2 buffers in play here - the BufferedOutputStream prevents thrift
from
+      // causing a call to deflate() on every encoded primitive. The DeflaterOutputStream
buffer
+      // allows the underlying Deflater to operate on a larger chunk at a time without stopping
to
+      // copy the intermediate compressed output to outBytes.
+      // See http://bugs.java.com/bugdatabase/view_bug.do?bug_id=4986239
+      TTransport transport = new TIOStreamTransport(
+          new BufferedOutputStream(
+              new DeflaterOutputStream(outBytes, new Deflater(), DEFLATER_BUFFER_SIZE),
+              DEFLATER_BUFFER_SIZE));
       TProtocol protocol = PROTOCOL_FACTORY.getProtocol(transport);
       tBase.write(protocol);
       transport.close();


Mime
View raw message