commons-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bode...@apache.org
Subject commons-compress git commit: allow LZ77 tuning in framed snappy/lz4 streams as well
Date Sun, 26 Mar 2017 16:56:00 GMT
Repository: commons-compress
Updated Branches:
  refs/heads/master 043f42b65 -> ad661a8d3


allow LZ77 tuning in framed snappy/lz4 streams as well


Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/ad661a8d
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/ad661a8d
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/ad661a8d

Branch: refs/heads/master
Commit: ad661a8d347c1c0ff1b9264cf0f8c89de7057a69
Parents: 043f42b
Author: Stefan Bodewig <bodewig@apache.org>
Authored: Sun Mar 26 18:55:34 2017 +0200
Committer: Stefan Bodewig <bodewig@apache.org>
Committed: Sun Mar 26 18:55:34 2017 +0200

----------------------------------------------------------------------
 .../lz4/FramedLZ4CompressorOutputStream.java    | 36 +++++++++++++++++++-
 .../FramedSnappyCompressorOutputStream.java     | 16 ++++++++-
 .../lz4/FramedLZ4CompressorRoundtripTest.java   |  4 +++
 3 files changed, 54 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/commons-compress/blob/ad661a8d/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
index 3ebcaa0..c08914f 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
@@ -88,6 +88,7 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream
{
     public static class Parameters {
         private final BlockSize blockSize;
         private final boolean withContentChecksum, withBlockChecksum, withBlockDependency;
+        private final org.apache.commons.compress.compressors.lz77support.Parameters lz77params;
 
         /**
          * The default parameters of 4M block size, enabled content
@@ -107,6 +108,16 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream
{
             this(blockSize, true, false, false);
         }
         /**
+         * Sets up custom a custom block size for the LZ4 stream but
+         * otherwise uses the defaults of enabled content checksum,
+         * disabled block checksums and independent blocks.
+         * @param blockSize the size of a single block.
+         */
+        public Parameters(BlockSize blockSize,
+            org.apache.commons.compress.compressors.lz77support.Parameters lz77params) {
+            this(blockSize, true, false, false, lz77params);
+        }
+        /**
          * Sets up custom parameters for the LZ4 stream.
          * @param blockSize the size of a single block.
          * @param withContentChecksum whether to write a content checksum
@@ -120,10 +131,33 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream
{
          */
         public Parameters(BlockSize blockSize, boolean withContentChecksum, boolean withBlockChecksum,
             boolean withBlockDependency) {
+            this(blockSize, withContentChecksum, withBlockChecksum, withBlockDependency,
+                 BlockLZ4CompressorOutputStream.createParameterBuilder().build());
+        }
+
+        /**
+         * Sets up custom parameters for the LZ4 stream.
+         * @param blockSize the size of a single block.
+         * @param withContentChecksum whether to write a content checksum
+         * @param withBlockChecksum whether to write a block checksum.
+         * Note that block checksums are not supported by the lz4
+         * command line utility
+         * @param withBlockDependency whether a block may depend on
+         * the content of a previous block. Enabling this may improve
+         * compression ratio but makes it impossible to decompress the
+         * output in parallel.
+         * @param lz77 parameters parameters used to fine-tune
+         * compression, in particular to balance compression ration vs
+         * compression speed.
+         */
+        public Parameters(BlockSize blockSize, boolean withContentChecksum, boolean withBlockChecksum,
+                boolean withBlockDependency,
+                org.apache.commons.compress.compressors.lz77support.Parameters lz77params)
{
             this.blockSize = blockSize;
             this.withContentChecksum = withContentChecksum;
             this.withBlockChecksum = withBlockChecksum;
             this.withBlockDependency = withBlockDependency;
+            this.lz77params = lz77params;
         }
 
         @Override
@@ -231,7 +265,7 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream
{
     private void flushBlock() throws IOException {
         final boolean withBlockDependency = params.withBlockDependency;
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        try (BlockLZ4CompressorOutputStream o = new BlockLZ4CompressorOutputStream(baos))
{
+        try (BlockLZ4CompressorOutputStream o = new BlockLZ4CompressorOutputStream(baos,
params.lz77params)) {
             if (withBlockDependency) {
                 o.prefill(blockDependencyBuffer, blockDependencyBuffer.length - collectedBlockDependencyBytes,
                     collectedBlockDependencyBytes);

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/ad661a8d/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java
b/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java
index e4d2a5e..fdb5699 100644
--- a/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorOutputStream.java
@@ -23,6 +23,7 @@ import java.io.IOException;
 import java.io.OutputStream;
 
 import org.apache.commons.compress.compressors.CompressorOutputStream;
+import org.apache.commons.compress.compressors.lz77support.Parameters;
 import org.apache.commons.compress.utils.ByteUtils;
 
 /**
@@ -42,6 +43,7 @@ public class FramedSnappyCompressorOutputStream extends CompressorOutputStream
{
     private static final int MAX_COMPRESSED_BUFFER_SIZE = 1 << 16;
 
     private final OutputStream out;
+    private final Parameters params;
     private final PureJavaCrc32C checksum = new PureJavaCrc32C();
     // used in one-arg write method
     private final byte[] oneByte = new byte[1];
@@ -57,7 +59,19 @@ public class FramedSnappyCompressorOutputStream extends CompressorOutputStream
{
      * @throws IOException if writing the signature fails
      */
     public FramedSnappyCompressorOutputStream(final OutputStream out) throws IOException
{
+        this(out, SnappyCompressorOutputStream.createParameterBuilder(SnappyCompressorInputStream.DEFAULT_BLOCK_SIZE)
+             .build());
+    }
+
+    /**
+     * Constructs a new output stream that compresses
+     * snappy-framed-compressed data to the specified output stream.
+     * @param out the OutputStream to which to write the compressed data
+     * @throws IOException if writing the signature fails
+     */
+    public FramedSnappyCompressorOutputStream(final OutputStream out, Parameters params)
throws IOException {
         this.out = out;
+        this.params = params;
         consumer = new ByteUtils.OutputStreamByteConsumer(out);
         out.write(FramedSnappyCompressorInputStream.SZ_SIGNATURE);
     }
@@ -104,7 +118,7 @@ public class FramedSnappyCompressorOutputStream extends CompressorOutputStream
{
     private void flushBuffer() throws IOException {
         out.write(FramedSnappyCompressorInputStream.COMPRESSED_CHUNK_TYPE);
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        try (OutputStream o = new SnappyCompressorOutputStream(baos, currentIndex)) {
+        try (OutputStream o = new SnappyCompressorOutputStream(baos, currentIndex, params))
{
             o.write(buffer, 0, currentIndex);
         }
         byte[] b = baos.toByteArray();

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/ad661a8d/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java
b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java
index d54fafe..ea63c1d 100644
--- a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java
+++ b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java
@@ -52,6 +52,10 @@ public final class FramedLZ4CompressorRoundtripTest extends AbstractTestCase
{
             // small blocksize (so we get enough blocks) and enabled block dependency, otherwise
defaults
             new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.K64,
                 true, false, true) },
+            // default, tuned for speed
+            new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.M4,
+                true, false, false, BlockLZ4CompressorOutputStream.createParameterBuilder()
+                    .tunedForSpeed().build()) },
         });
     }
 


Mime
View raw message