commons-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bode...@apache.org
Subject commons-compress git commit: COMPRESS-271 frame header checksums
Date Wed, 25 Jan 2017 05:36:34 GMT
Repository: commons-compress
Updated Branches:
  refs/heads/master b5d6f1f62 -> 76d913a3c


COMPRESS-271 frame header checksums


Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/76d913a3
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/76d913a3
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/76d913a3

Branch: refs/heads/master
Commit: 76d913a3cd2224a1e6a13af77f77709ee1b265c6
Parents: b5d6f1f
Author: Stefan Bodewig <bodewig@apache.org>
Authored: Wed Jan 25 06:36:12 2017 +0100
Committer: Stefan Bodewig <bodewig@apache.org>
Committed: Wed Jan 25 06:36:12 2017 +0100

----------------------------------------------------------------------
 .../lz4/FramedLZ4CompressorInputStream.java     | 20 +++++++++++++---
 .../lz4/FramedLZ4CompressorOutputStream.java    | 18 ++++++++++----
 .../lz4/FramedLZ4CompressorInputStreamTest.java | 25 ++++++++++++++++----
 3 files changed, 51 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/commons-compress/blob/76d913a3/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
index a7f4a4d..7cbd688 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
@@ -80,6 +80,9 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream
{
     private InputStream currentBlock;
     private boolean endReached, inUncompressed;
 
+    // used for frame header checksum and content checksum, if present
+    private final XXHash32 contentHash = new XXHash32();
+
     /**
      * Creates a new input stream that decompresses streams compressed
      * using the LZ4 frame format.
@@ -139,6 +142,7 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream
{
         if (flags == -1) {
             throw new IOException("Premature end of stream while reading frame flags");
         }
+        contentHash.update(flags);
         if ((flags & VERSION_MASK) != SUPPORTED_VERSION) {
             throw new IOException("Unsupported version " + (flags >> 6));
         }
@@ -148,19 +152,29 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream
{
         expectBlockChecksum = (flags & BLOCK_CHECKSUM_MASK) != 0;
         expectContentSize = (flags & CONTENT_SIZE_MASK) != 0;
         expectContentChecksum = (flags & CONTENT_CHECKSUM_MASK) != 0;
-        if (readOneByte() == -1) { // max size is irrelevant for this implementation
+        int bdByte = readOneByte();
+        if (bdByte == -1) { // max size is irrelevant for this implementation
             throw new IOException("Premature end of stream while reading frame BD byte");
         }
+        contentHash.update(bdByte);
         if (expectContentSize) { // for now we don't care, contains the uncompressed size
-            int skipped = (int) IOUtils.skip(in, 8);
+            byte[] contentSize = new byte[8];
+            int skipped = (int) IOUtils.readFully(in, contentSize);
             count(skipped);
             if (8 != skipped) {
                 throw new IOException("Premature end of stream while reading content size");
             }
+            contentHash.update(contentSize, 0, contentSize.length);
         }
-        if (readOneByte() == -1) { // partial hash of header. not supported, yet
+        int headerHash = readOneByte();
+        if (headerHash == -1) { // partial hash of header.
             throw new IOException("Premature end of stream while reading frame header checksum");
         }
+        int expectedHash = (int) ((contentHash.getValue() >> 8) & 0xff);
+        contentHash.reset();
+        if (headerHash != expectedHash) {
+            throw new IOException("frame header checksum mismatch.");
+        }
     }
 
     private void nextBlock() throws IOException {

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/76d913a3/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
index f240e0b..e0622e1 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
@@ -40,7 +40,7 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream
{
     /*
      * TODO before releasing 1.14:
      *
-     * + xxhash32 checksum creation for headers, content, blocks
+     * + xxhash32 checksum creation for content, blocks
      * + block dependence
      */
 
@@ -57,6 +57,9 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream
{
     private boolean finished = false;
     private int currentIndex = 0;
 
+    // used for frame header checksum and content checksum, if present
+    private final XXHash32 contentHash = new XXHash32();
+
     /**
      * Constructs a new output stream that compresses data using the
      * LZ4 frame format using the default block size of 4MB.
@@ -129,10 +132,15 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream
{
     }
 
     private void writeFrameDescriptor() throws IOException {
-        out.write(FramedLZ4CompressorInputStream.SUPPORTED_VERSION
-            | FramedLZ4CompressorInputStream.BLOCK_INDEPENDENCE_MASK);
-        out.write(BLOCK_SIZES.indexOf(blockData.length) << 4);
-        out.write(0); // TODO header checksum
+        int flags = FramedLZ4CompressorInputStream.SUPPORTED_VERSION
+            | FramedLZ4CompressorInputStream.BLOCK_INDEPENDENCE_MASK;
+        out.write(flags);
+        contentHash.update(flags);
+        int bd = BLOCK_SIZES.indexOf(blockData.length) << 4;
+        out.write(bd);
+        contentHash.update(bd);
+        out.write((int) ((contentHash.getValue() >> 8) & 0xff));
+        contentHash.reset();
     }
 
     private void flushBlock() throws IOException {

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/76d913a3/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
index 3727518..0893fbe 100644
--- a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
+++ b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
@@ -166,12 +166,29 @@ public final class FramedLZ4CompressorInputStreamTest
     }
 
     @Test
+    public void rejectsFileWithBadHeaderChecksum() throws IOException {
+        byte[] input = new byte[] {
+            4, 0x22, 0x4d, 0x18, // signature
+            0x64, // flag - Version 01, block independent, no block checksum, no content
size, with content checksum
+            0x70, // block size 4MB
+            0,
+        };
+        try {
+            try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input)))
{
+                fail("expected exception");
+            }
+        } catch (IOException ex) {
+            assertThat(ex.getMessage(), containsString("header checksum mismatch"));
+        }
+    }
+
+    @Test
     public void readsUncompressedBlocks() throws IOException {
         byte[] input = new byte[] {
             4, 0x22, 0x4d, 0x18, // signature
             0x60, // flag - Version 01, block independent, no block checksum, no content
size, no content checksum
             0x70, // block size 4MB
-            0x00, // checksum, revisit once it gets validated
+            115, // checksum
             13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
             'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
             0, 0, 0, 0, // empty block marker
@@ -190,7 +207,7 @@ public final class FramedLZ4CompressorInputStreamTest
             4, 0x22, 0x4d, 0x18, // signature
             0x60, // flag - Version 01, block independent, no block checksum, no content
size, no content checksum
             0x70, // block size 4MB
-            0x00, // checksum, revisit once it gets validated
+            115, // checksum
             13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
             'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
             0, 0, 0, 0, // empty block marker
@@ -207,7 +224,7 @@ public final class FramedLZ4CompressorInputStreamTest
             4, 0x22, 0x4d, 0x18, // signature
             0x70, // flag - Version 01, block independent, with block checksum, no content
size, no content checksum
             0x70, // block size 4MB
-            0x00, // checksum, revisit once it gets validated
+            114, // checksum
             13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
             'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
         };
@@ -227,7 +244,7 @@ public final class FramedLZ4CompressorInputStreamTest
             4, 0x22, 0x4d, 0x18, // signature
             0x64, // flag - Version 01, block independent, no block checksum, no content
size, with content checksum
             0x70, // block size 4MB
-            0x00, // checksum, revisit once it gets validated
+            (byte) 185, // checksum
             13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set
             'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content
             0, 0, 0, 0, // empty block marker


Mime
View raw message