commons-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bode...@apache.org
Subject commons-compress git commit: COMPRESS-271 support for decompressConcatenated in lz4 frame input
Date Sun, 05 Feb 2017 11:55:52 GMT
Repository: commons-compress
Updated Branches:
  refs/heads/master 3af95ce3a -> 25ecc9b17


COMPRESS-271 support for decompressConcatenated in lz4 frame input


Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/25ecc9b1
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/25ecc9b1
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/25ecc9b1

Branch: refs/heads/master
Commit: 25ecc9b177ef21bcc35a82776b71cbdae4a7f708
Parents: 3af95ce
Author: Stefan Bodewig <bodewig@apache.org>
Authored: Sun Feb 5 12:55:12 2017 +0100
Committer: Stefan Bodewig <bodewig@apache.org>
Committed: Sun Feb 5 12:55:12 2017 +0100

----------------------------------------------------------------------
 .../compressors/CompressorStreamFactory.java    |   2 +-
 .../lz4/FramedLZ4CompressorInputStream.java     |  45 ++++++--
 .../lz4/FramedLZ4CompressorInputStreamTest.java | 105 +++++++++++++++++++
 3 files changed, 143 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/commons-compress/blob/25ecc9b1/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
index 29ba9fd..4f0685b 100644
--- a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
+++ b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
@@ -518,7 +518,7 @@ public class CompressorStreamFactory implements CompressorStreamProvider
{
             }
 
             if (LZ4_FRAMED.equalsIgnoreCase(name)) {
-                return new FramedLZ4CompressorInputStream(in);
+                return new FramedLZ4CompressorInputStream(in, actualDecompressConcatenated);
             }
 
         } catch (final IOException e) {

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/25ecc9b1/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
index 418f320..e597b5a 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
@@ -40,9 +40,7 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream
{
     /*
      * TODO before releasing 1.14:
      *
-     * + xxhash32 checksum validation
      * + skippable frames
-     * + decompressConcatenated
      * + block dependence
      */
 
@@ -71,6 +69,7 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream
{
     };
 
     private final InputStream in;
+    private final boolean decompressConcatenated;
 
     private boolean expectBlockChecksum;
     private boolean expectContentSize;
@@ -84,15 +83,29 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream
{
 
     /**
      * Creates a new input stream that decompresses streams compressed
-     * using the LZ4 frame format.
+     * using the LZ4 frame format and stops after decompressing the
+     * first frame.
      * @param in  the InputStream from which to read the compressed data
      * @throws IOException if reading fails
      */
     public FramedLZ4CompressorInputStream(InputStream in) throws IOException {
+        this(in, false);
+    }
+
+    /**
+     * Creates a new input stream that decompresses streams compressed
+     * using the LZ4 frame format.
+     * @param in  the InputStream from which to read the compressed data
+     * @param decompressConcatenated if true, decompress until the end
+     *          of the input; if false, stop after the first LZ4 frame
+     *          and leave the input position to point to the next byte
+     *          after the frame stream
+     * @throws IOException if reading fails
+     */
+    public FramedLZ4CompressorInputStream(InputStream in, boolean decompressConcatenated)
throws IOException {
         this.in = in;
-        readSignature();
-        readFrameDescriptor();
-        nextBlock();
+        this.decompressConcatenated = decompressConcatenated;
+        init(true);
     }
 
     /** {@inheritDoc} */
@@ -130,13 +143,25 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream
{
         return r;
     }
 
-    private void readSignature() throws IOException {
+    private void init(boolean firstFrame) throws IOException {
+        if (readSignature(firstFrame)) {
+            readFrameDescriptor();
+            nextBlock();
+        }
+    }
+
+    private boolean readSignature(boolean firstFrame) throws IOException {
         final byte[] b = new byte[4];
         final int read = IOUtils.readFully(in, b);
         count(read);
+        if (4 != read && !firstFrame) {
+            endReached = true;
+            return false;
+        }
         if (4 != read || !matches(b, 4)) {
             throw new IOException("Not a LZ4 frame stream");
         }
+        return true;
     }
 
     private void readFrameDescriptor() throws IOException {
@@ -185,8 +210,12 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream
{
         boolean uncompressed = (len & UNCOMPRESSED_FLAG_MASK) != 0;
         int realLen = (int) (len & (~UNCOMPRESSED_FLAG_MASK));
         if (realLen == 0) {
-            endReached = true;
             verifyContentChecksum();
+            if (!decompressConcatenated) {
+                endReached = true;
+            } else {
+                init(false);
+            }
             return;
         }
         InputStream capped = new BoundedInputStream(in, realLen);

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/25ecc9b1/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
index 3f5c701..0363e5e 100644
--- a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
+++ b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
@@ -27,6 +27,7 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.InputStream;
 import java.io.IOException;
+import java.util.Arrays;
 
 import org.apache.commons.compress.AbstractTestCase;
 import org.apache.commons.compress.compressors.CompressorStreamFactory;
@@ -82,6 +83,86 @@ public final class FramedLZ4CompressorInputStreamTest
         }
     }
 
+    @Test
+    public void readBlaLz4WithDecompressConcatenated() throws IOException {
+        try (InputStream a = new FramedLZ4CompressorInputStream(new FileInputStream(getFile("bla.tar.lz4")),
true);
+            FileInputStream e = new FileInputStream(getFile("bla.tar"))) {
+            byte[] expected = IOUtils.toByteArray(e);
+            byte[] actual = IOUtils.toByteArray(a);
+            assertArrayEquals(expected, actual);
+        }
+    }
+
+    @Test
+    public void readDoubledBlaLz4WithDecompressConcatenatedTrue() throws Exception {
+        readDoubledBlaLz4(new StreamWrapper() {
+                public InputStream wrap(InputStream in) throws Exception {
+                    return new FramedLZ4CompressorInputStream(in, true);
+                }
+            }, true);
+    }
+
+    @Test
+    public void readDoubledBlaLz4WithDecompressConcatenatedFalse() throws Exception {
+        readDoubledBlaLz4(new StreamWrapper() {
+                public InputStream wrap(InputStream in) throws Exception {
+                    return new FramedLZ4CompressorInputStream(in, false);
+                }
+            }, false);
+    }
+
+    @Test
+    public void readDoubledBlaLz4WithoutExplicitDecompressConcatenated() throws Exception
{
+        readDoubledBlaLz4(new StreamWrapper() {
+                public InputStream wrap(InputStream in) throws Exception {
+                    return new FramedLZ4CompressorInputStream(in);
+                }
+            }, false);
+    }
+
+    @Test
+    public void readBlaLz4ViaFactoryWithDecompressConcatenated() throws Exception {
+        try (InputStream a = new CompressorStreamFactory()
+                 .createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(),
+                                              new FileInputStream(getFile("bla.tar.lz4")),
+                                              true);
+            FileInputStream e = new FileInputStream(getFile("bla.tar"))) {
+            byte[] expected = IOUtils.toByteArray(e);
+            byte[] actual = IOUtils.toByteArray(a);
+            assertArrayEquals(expected, actual);
+        }
+    }
+
+    @Test
+    public void readDoubledBlaLz4ViaFactoryWithDecompressConcatenatedTrue() throws Exception
{
+        readDoubledBlaLz4(new StreamWrapper() {
+                public InputStream wrap(InputStream in) throws Exception {
+                    return new CompressorStreamFactory()
+                        .createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(),
in, true);
+                }
+            }, true);
+    }
+
+    @Test
+    public void readDoubledBlaLz4ViaFactoryWithDecompressConcatenatedFalse() throws Exception
{
+        readDoubledBlaLz4(new StreamWrapper() {
+                public InputStream wrap(InputStream in) throws Exception {
+                    return new CompressorStreamFactory()
+                        .createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(),
in, false);
+                }
+            }, false);
+    }
+
+    @Test
+    public void readDoubledBlaLz4ViaFactoryWithoutExplicitDecompressConcatenated() throws
Exception {
+        readDoubledBlaLz4(new StreamWrapper() {
+                public InputStream wrap(InputStream in) throws Exception {
+                    return new CompressorStreamFactory()
+                        .createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(),
in);
+                }
+            }, false);
+    }
+
     @Test(expected = IOException.class)
     public void rejectsNonLZ4Stream() throws IOException {
         try (InputStream a = new FramedLZ4CompressorInputStream(new FileInputStream(getFile("bla.tar"))))
{
@@ -280,4 +361,28 @@ public final class FramedLZ4CompressorInputStreamTest
             assertThat(ex.getMessage(), containsString("content checksum mismatch"));
         }
     }
+
+    interface StreamWrapper {
+        InputStream wrap(InputStream in) throws Exception;
+    }
+
+    private void readDoubledBlaLz4(StreamWrapper wrapper, boolean expectDuplicateOutput)
throws Exception {
+        byte[] singleInput;
+        try (InputStream i = new FileInputStream(getFile("bla.tar.lz4"))) {
+            singleInput = IOUtils.toByteArray(i);
+        }
+        byte[] input = duplicate(singleInput);
+        try (InputStream a = wrapper.wrap(new ByteArrayInputStream(input));
+            FileInputStream e = new FileInputStream(getFile("bla.tar"))) {
+            byte[] expected = IOUtils.toByteArray(e);
+            byte[] actual = IOUtils.toByteArray(a);
+            assertArrayEquals(expectDuplicateOutput ? duplicate(expected) : expected, actual);
+        }
+    }
+
+    private static byte[] duplicate(byte[] from) {
+        byte[] to = Arrays.copyOf(from, 2 * from.length);
+        System.arraycopy(from, 0, to, from.length, from.length);
+        return to;
+    }
 }


Mime
View raw message