commons-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bode...@apache.org
Subject commons-compress git commit: COMPRESS-271 support for skipping skippable frames in lz4
Date Sun, 05 Feb 2017 14:21:12 GMT
Repository: commons-compress
Updated Branches:
  refs/heads/master 2239893ab -> 68bc083e6


COMPRESS-271 support for skipping skippable frames in lz4


Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/68bc083e
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/68bc083e
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/68bc083e

Branch: refs/heads/master
Commit: 68bc083e6ba002f7c997f9781f0e24a80d8a3274
Parents: 2239893
Author: Stefan Bodewig <bodewig@apache.org>
Authored: Sun Feb 5 15:20:37 2017 +0100
Committer: Stefan Bodewig <bodewig@apache.org>
Committed: Sun Feb 5 15:20:37 2017 +0100

----------------------------------------------------------------------
 .../lz4/FramedLZ4CompressorInputStream.java     |  56 ++++-
 .../lz4/FramedLZ4CompressorInputStreamTest.java | 214 +++++++++++++++++++
 2 files changed, 266 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/commons-compress/blob/68bc083e/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
index 115a2d6..2929bae 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
@@ -40,7 +40,6 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream
{
     /*
      * TODO before releasing 1.14:
      *
-     * + skippable frames
      * + block dependence
      */
 
@@ -48,6 +47,10 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream
{
     static final byte[] LZ4_SIGNATURE = new byte[] { //NOSONAR
         4, 0x22, 0x4d, 0x18
     };
+    private static final byte[] SKIPPABLE_FRAME_TRAILER = new byte[] {
+        0x2a, 0x4d, 0x18
+    };
+    private static final byte SKIPPABLE_FRAME_PREFIX_BYTE_MASK = 0x50;
 
     static final int VERSION_MASK = 0xC0;
     static final int SUPPORTED_VERSION = 0x40;
@@ -151,15 +154,25 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream
{
     }
 
     private boolean readSignature(boolean firstFrame) throws IOException {
+        String garbageMessage = firstFrame ? "Not a LZ4 frame stream" : "LZ4 frame stream
followed by garbage";
         final byte[] b = new byte[4];
-        final int read = IOUtils.readFully(in, b);
+        int read = IOUtils.readFully(in, b);
         count(read);
-        if (4 != read && !firstFrame) {
+        if (0 == read && !firstFrame) {
+            endReached = true;
+            return false;
+        }
+        if (4 != read) {
+            throw new IOException(garbageMessage);
+        }
+
+        read = skipSkippableFrame(b);
+        if (0 == read && !firstFrame) {
             endReached = true;
             return false;
         }
         if (4 != read || !matches(b, 4)) {
-            throw new IOException("Not a LZ4 frame stream");
+            throw new IOException(garbageMessage);
         }
         return true;
     }
@@ -281,6 +294,41 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream
{
         }
     }
 
+    private static boolean isSkippableFrameSignature(byte[] b) {
+        if ((b[0] & SKIPPABLE_FRAME_PREFIX_BYTE_MASK) != SKIPPABLE_FRAME_PREFIX_BYTE_MASK)
{
+            return false;
+        }
+        for (int i = 1; i < 4; i++) {
+            if (b[i] != SKIPPABLE_FRAME_TRAILER[i - 1]) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Skips over the contents of a skippable frame as well as
+     * skippable frames following it.
+     *
+     * <p>It then tries to read four more bytes which are supposed to
+     * hold an LZ4 signature and returns the number of bytes read
+     * while storing the bytes in the given array.</p>
+     */
+    private int skipSkippableFrame(byte[] b) throws IOException {
+        int read = 4;
+        while (read == 4 && isSkippableFrameSignature(b)) {
+            long len = ByteUtils.fromLittleEndian(supplier, 4);
+            long skipped = IOUtils.skip(in, len);
+            count(skipped);
+            if (len != skipped) {
+                throw new IOException("Premature end of stream while skipping frame");
+            }
+            read = IOUtils.readFully(in, b);
+            count(read);
+        }
+        return read;
+    }
+
     /**
      * Checks if the signature matches what is expected for a .lz4 file.
      *

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/68bc083e/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
index 0363e5e..9dda432 100644
--- a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
+++ b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
@@ -362,6 +362,220 @@ public final class FramedLZ4CompressorInputStreamTest
         }
     }
 
+    @Test
+    public void skipsOverSkippableFrames() throws IOException {
+        byte[] input = new byte[] {
+            4, 0x22, 0x4d, 0x18, // signature
+            0x60, // flag - Version 01, block independent, no block checksum, no content
size, no content checksum
+            0x70, // block size 4MB
+            115, // checksum
+            13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
+            'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
+            0, 0, 0, 0, // empty block marker
+            0x5f, 0x2a, 0x4d, 0x18, // skippable frame signature
+            2, 0, 0, 0, // skippable frame has length 2
+            1, 2, // content of skippable frame
+            4, 0x22, 0x4d, 0x18, // signature
+            0x60, // flag - Version 01, block independent, no block checksum, no content
size, no content checksum
+            0x70, // block size 4MB
+            115, // checksum
+            1, 0, 0, (byte) 0x80, // 1 bytes length and uncompressed bit set
+            '!', // content
+            0, 0, 0, 0, // empty block marker
+        };
+        try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input),
true)) {
+            byte[] actual = IOUtils.toByteArray(a);
+            assertArrayEquals(new byte[] {
+                    'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', '!'
+                }, actual);
+        }
+    }
+
+    @Test
+    public void skipsOverTrailingSkippableFrames() throws IOException {
+        byte[] input = new byte[] {
+            4, 0x22, 0x4d, 0x18, // signature
+            0x60, // flag - Version 01, block independent, no block checksum, no content
size, no content checksum
+            0x70, // block size 4MB
+            115, // checksum
+            13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
+            'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
+            0, 0, 0, 0, // empty block marker
+            0x51, 0x2a, 0x4d, 0x18, // skippable frame signature
+            2, 0, 0, 0, // skippable frame has length 2
+            1, 2, // content of skippable frame
+        };
+        try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input),
true)) {
+            byte[] actual = IOUtils.toByteArray(a);
+            assertArrayEquals(new byte[] {
+                    'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!'
+                }, actual);
+        }
+    }
+
+    @Test
+    public void rejectsSkippableFrameFollowedByJunk() throws IOException {
+        byte[] input = new byte[] {
+            4, 0x22, 0x4d, 0x18, // signature
+            0x60, // flag - Version 01, block independent, no block checksum, no content
size, no content checksum
+            0x70, // block size 4MB
+            115, // checksum
+            13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
+            'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
+            0, 0, 0, 0, // empty block marker
+            0x50, 0x2a, 0x4d, 0x18, // skippable frame signature
+            2, 0, 0, 0, // skippable frame has length 2
+            1, 2, // content of skippable frame
+            1, 0x22, 0x4d, 0x18, // bad signature
+        };
+        try {
+            try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input),
true)) {
+                IOUtils.toByteArray(a);
+                fail("expected exception");
+            }
+        } catch (IOException ex) {
+            assertThat(ex.getMessage(), containsString("garbage"));
+        }
+    }
+
+    @Test
+    public void rejectsSkippableFrameFollowedByTooFewBytes() throws IOException {
+        byte[] input = new byte[] {
+            4, 0x22, 0x4d, 0x18, // signature
+            0x60, // flag - Version 01, block independent, no block checksum, no content
size, no content checksum
+            0x70, // block size 4MB
+            115, // checksum
+            13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
+            'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
+            0, 0, 0, 0, // empty block marker
+            0x52, 0x2a, 0x4d, 0x18, // skippable frame signature
+            2, 0, 0, 0, // skippable frame has length 2
+            1, 2, // content of skippable frame
+            4, // too short for signature
+        };
+        try {
+            try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input),
true)) {
+                IOUtils.toByteArray(a);
+                fail("expected exception");
+            }
+        } catch (IOException ex) {
+            assertThat(ex.getMessage(), containsString("garbage"));
+        }
+    }
+
+    @Test
+    public void rejectsSkippableFrameWithPrematureEnd() throws IOException {
+        byte[] input = new byte[] {
+            4, 0x22, 0x4d, 0x18, // signature
+            0x60, // flag - Version 01, block independent, no block checksum, no content
size, no content checksum
+            0x70, // block size 4MB
+            115, // checksum
+            13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
+            'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
+            0, 0, 0, 0, // empty block marker
+            0x50, 0x2a, 0x4d, 0x18, // skippable frame signature
+            2, 0, 0, 0, // skippable frame has length 2
+            1, // content of skippable frame (should be two bytes)
+        };
+        try {
+            try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input),
true)) {
+                IOUtils.toByteArray(a);
+                fail("expected exception");
+            }
+        } catch (IOException ex) {
+            assertThat(ex.getMessage(), containsString("Premature end of stream while skipping
frame"));
+        }
+    }
+
+    @Test
+    public void rejectsSkippableFrameWithPrematureEndInLengthBytes() throws IOException {
+        byte[] input = new byte[] {
+            4, 0x22, 0x4d, 0x18, // signature
+            0x60, // flag - Version 01, block independent, no block checksum, no content
size, no content checksum
+            0x70, // block size 4MB
+            115, // checksum
+            13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
+            'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
+            0, 0, 0, 0, // empty block marker
+            0x55, 0x2a, 0x4d, 0x18, // skippable frame signature
+            2, 0, 0, // should be four byte length
+        };
+        try {
+            try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input),
true)) {
+                IOUtils.toByteArray(a);
+                fail("expected exception");
+            }
+        } catch (IOException ex) {
+            assertThat(ex.getMessage(), containsString("premature end of data"));
+        }
+    }
+
+    @Test
+    public void rejectsSkippableFrameWithBadSignatureTrailer() throws IOException {
+        byte[] input = new byte[] {
+            4, 0x22, 0x4d, 0x18, // signature
+            0x60, // flag - Version 01, block independent, no block checksum, no content
size, no content checksum
+            0x70, // block size 4MB
+            115, // checksum
+            13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
+            'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
+            0, 0, 0, 0, // empty block marker
+            0x51, 0x2a, 0x4d, 0x17, // broken skippable frame signature
+        };
+        try {
+            try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input),
true)) {
+                IOUtils.toByteArray(a);
+                fail("expected exception");
+            }
+        } catch (IOException ex) {
+            assertThat(ex.getMessage(), containsString("garbage"));
+        }
+    }
+
+    @Test
+    public void rejectsSkippableFrameWithBadSignaturePrefix() throws IOException {
+        byte[] input = new byte[] {
+            4, 0x22, 0x4d, 0x18, // signature
+            0x60, // flag - Version 01, block independent, no block checksum, no content
size, no content checksum
+            0x70, // block size 4MB
+            115, // checksum
+            13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
+            'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
+            0, 0, 0, 0, // empty block marker
+            0x60, 0x2a, 0x4d, 0x18, // broken skippable frame signature
+        };
+        try {
+            try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input),
true)) {
+                IOUtils.toByteArray(a);
+                fail("expected exception");
+            }
+        } catch (IOException ex) {
+            assertThat(ex.getMessage(), containsString("garbage"));
+        }
+    }
+
+    @Test
+    public void rejectsTrailingBytesAfterValidFrame() throws IOException {
+        byte[] input = new byte[] {
+            4, 0x22, 0x4d, 0x18, // signature
+            0x60, // flag - Version 01, block independent, no block checksum, no content
size, no content checksum
+            0x70, // block size 4MB
+            115, // checksum
+            13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
+            'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
+            0, 0, 0, 0, // empty block marker
+            0x56, 0x2a, 0x4d, // too short for any signature
+        };
+        try {
+            try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input),
true)) {
+                IOUtils.toByteArray(a);
+                fail("expected exception");
+            }
+        } catch (IOException ex) {
+            assertThat(ex.getMessage(), containsString("garbage"));
+        }
+    }
+
     interface StreamWrapper {
         InputStream wrap(InputStream in) throws Exception;
     }


Mime
View raw message