commons-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bode...@apache.org
Subject [2/2] commons-compress git commit: COMPRESS-271 add read-support for LZ4 with block dependency
Date Sun, 05 Feb 2017 18:09:07 GMT
COMPRESS-271 add read-support for LZ4 with block dependency


Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/8c75380e
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/8c75380e
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/8c75380e

Branch: refs/heads/master
Commit: 8c75380e482d7c9945098a258ebba4fe2eba4264
Parents: dc45445
Author: Stefan Bodewig <bodewig@apache.org>
Authored: Sun Feb 5 19:08:31 2017 +0100
Committer: Stefan Bodewig <bodewig@apache.org>
Committed: Sun Feb 5 19:08:31 2017 +0100

----------------------------------------------------------------------
 .../lz4/FramedLZ4CompressorInputStream.java     | 40 ++++++++++++++++----
 .../AbstractLZ77CompressorInputStream.java      | 21 ++++++++++
 .../lz4/FramedLZ4CompressorInputStreamTest.java | 10 +++++
 3 files changed, 63 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/commons-compress/blob/8c75380e/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
index 2929bae..9b47d7b 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java
@@ -37,11 +37,6 @@ import org.apache.commons.compress.utils.IOUtils;
  * @NotThreadSafe
  */
 public class FramedLZ4CompressorInputStream extends CompressorInputStream {
-    /*
-     * TODO before releasing 1.14:
-     *
-     * + block dependence
-     */
 
     // used by FramedLZ4CompressorOutputStream as well
     static final byte[] LZ4_SIGNATURE = new byte[] { //NOSONAR
@@ -75,6 +70,7 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream
{
     private final boolean decompressConcatenated;
 
     private boolean expectBlockChecksum;
+    private boolean expectBlockDependency;
     private boolean expectContentSize;
     private boolean expectContentChecksum;
 
@@ -84,6 +80,9 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream
{
     // used for frame header checksum and content checksum, if present
     private final XXHash32 contentHash = new XXHash32();
 
+    // only created if the frame doesn't set the block independence flag
+    private byte[] blockDependencyBuffer;
+
     /**
      * Creates a new input stream that decompresses streams compressed
      * using the LZ4 frame format and stops after decompressing the
@@ -140,6 +139,9 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream
{
                 r = readOnce(b, off, len);
             }
         }
+        if (expectBlockDependency) {
+            appendToBlockDependencyBuffer(b, off, r);
+        }
         if (expectContentChecksum && r != -1) {
             contentHash.update(b, off, r);
         }
@@ -186,8 +188,13 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream
{
         if ((flags & VERSION_MASK) != SUPPORTED_VERSION) {
             throw new IOException("Unsupported version " + (flags >> 6));
         }
-        if ((flags & BLOCK_INDEPENDENCE_MASK) == 0) {
-            throw new IOException("Block dependence is not supported");
+        expectBlockDependency = (flags & BLOCK_INDEPENDENCE_MASK) == 0;
+        if (expectBlockDependency) {
+            if (blockDependencyBuffer == null) {
+                blockDependencyBuffer = new byte[BlockLZ4CompressorInputStream.WINDOW_SIZE];
+            }
+        } else {
+            blockDependencyBuffer = null;
         }
         expectBlockChecksum = (flags & BLOCK_CHECKSUM_MASK) != 0;
         expectContentSize = (flags & CONTENT_SIZE_MASK) != 0;
@@ -237,7 +244,11 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream
{
             currentBlock = capped;
         } else {
             inUncompressed = false;
-            currentBlock = new BlockLZ4CompressorInputStream(capped);
+            BlockLZ4CompressorInputStream s = new BlockLZ4CompressorInputStream(capped);
+            if (expectBlockDependency) {
+                s.prefill(blockDependencyBuffer);
+            }
+            currentBlock = s;
         }
     }
 
@@ -329,6 +340,19 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream
{
         return read;
     }
 
+    private void appendToBlockDependencyBuffer(final byte[] b, final int off, int len) {
+        len = Math.min(len, blockDependencyBuffer.length);
+        if (len > 0) {
+            int keep = blockDependencyBuffer.length - len;
+            if (keep > 0) {
+                // move last keep bytes towards the start of the buffer
+                System.arraycopy(blockDependencyBuffer, len, blockDependencyBuffer, 0, keep);
+            }
+            // append new data
+            System.arraycopy(b, off, blockDependencyBuffer, keep, len);
+        }
+    }
+
     /**
      * Checks if the signature matches what is expected for a .lz4 file.
      *

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/8c75380e/src/main/java/org/apache/commons/compress/compressors/lz77support/AbstractLZ77CompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz77support/AbstractLZ77CompressorInputStream.java
b/src/main/java/org/apache/commons/compress/compressors/lz77support/AbstractLZ77CompressorInputStream.java
index 49372e4..a406b85 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lz77support/AbstractLZ77CompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lz77support/AbstractLZ77CompressorInputStream.java
@@ -120,6 +120,27 @@ public abstract class AbstractLZ77CompressorInputStream extends CompressorInputS
     }
 
     /**
+     * Adds some initial data to fill the window with.
+     *
+     * <p>This is used if the stream has been cut into blocks and
+     * back-references of one block may refer to data of the previous
+     * block(s). One such example is the LZ4 frame format using block
+     * dependency.</p>
+     *
+     * @param data the data to fill the window with.
+     * @throws IllegalStateException if the stream has already started to read data
+     */
+    public void prefill(byte[] data) {
+        if (writeIndex != 0) {
+            throw new IllegalStateException("the stream has already been read from, can't
prefill anymore");
+        }
+        int len = Math.min(windowSize, data.length);
+        System.arraycopy(data, data.length - len, buf, 0, len);
+        writeIndex += len;
+        readIndex += len;
+    }
+
+    /**
      * Used by subclasses to signal the next block contains the given
      * amount of literal data.
      * @param length the length of the block

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/8c75380e/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
index 9dda432..a1e28b5 100644
--- a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
+++ b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java
@@ -163,6 +163,16 @@ public final class FramedLZ4CompressorInputStreamTest
             }, false);
     }
 
+    @Test
+    public void readBlaDumpLz4() throws IOException {
+        try (InputStream a = new FramedLZ4CompressorInputStream(new FileInputStream(getFile("bla.dump.lz4")));
+            FileInputStream e = new FileInputStream(getFile("bla.dump"))) {
+            byte[] expected = IOUtils.toByteArray(e);
+            byte[] actual = IOUtils.toByteArray(a);
+            assertArrayEquals(expected, actual);
+        }
+    }
+
     @Test(expected = IOException.class)
     public void rejectsNonLZ4Stream() throws IOException {
         try (InputStream a = new FramedLZ4CompressorInputStream(new FileInputStream(getFile("bla.tar"))))
{


Mime
View raw message