Mailing-List: contact commits-help@commons.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@commons.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: bodewig@apache.org
To: commits@commons.apache.org
Message-Id: <7d1dc80f05b94d84bd1563952a8d5554@git.apache.org>
Subject: commons-compress git commit: COMPRESS-271 non-functional skeleton of
 LZ4 block output stream
Date: Wed, 18 Jan 2017 05:42:07 +0000 (UTC)
archived-at: Wed, 18 Jan 2017 05:42:09 -0000

Repository: commons-compress
Updated Branches:
  refs/heads/master b8939bea3 -> 74b38da45


COMPRESS-271 non-functional skeleton of LZ4 block output stream


Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/74b38da4
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/74b38da4
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/74b38da4

Branch: refs/heads/master
Commit: 74b38da45a6601a5f4d8c38e417c547024abaf4c
Parents: b8939be
Author: Stefan Bodewig <bodewig@apache.org>
Authored: Wed Jan 18 06:41:23 2017 +0100
Committer: Stefan Bodewig <bodewig@apache.org>
Committed: Wed Jan 18 06:41:23 2017 +0100

----------------------------------------------------------------------
 .../lz4/BlockLZ4CompressorInputStream.java      |   8 +-
 .../lz4/BlockLZ4CompressorOutputStream.java     | 144 +++++++++++++++++++
 2 files changed, 148 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/commons-compress/blob/74b38da4/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorInputStream.java
index bdefcb2..a93f10d 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorInputStream.java
@@ -32,10 +32,10 @@ import org.apache.commons.compress.utils.ByteUtils;
  */
 public class BlockLZ4CompressorInputStream extends AbstractLZ77CompressorInputStream {
 
-    private static final int WINDOW_SIZE = 1 << 16;
-    private static final int SIZE_BITS = 4;
-    private static final int COPY_SIZE_MASK = (1 << SIZE_BITS) - 1;
-    private static final int LITERAL_SIZE_MASK = COPY_SIZE_MASK << SIZE_BITS;
+    static final int WINDOW_SIZE = 1 << 16;
+    static final int SIZE_BITS = 4;
+    static final int COPY_SIZE_MASK = (1 << SIZE_BITS) - 1;
+    static final int LITERAL_SIZE_MASK = COPY_SIZE_MASK << SIZE_BITS;
 
     /** Copy-size part of the block starting byte. */
     private int nextCopySize;

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/74b38da4/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java
new file mode 100644
index 0000000..79e1310
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress.compressors.lz4;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.commons.compress.compressors.CompressorOutputStream;
+import org.apache.commons.compress.compressors.lz77support.LZ77Compressor;
+import org.apache.commons.compress.compressors.lz77support.Parameters;
+import org.apache.commons.compress.utils.ByteUtils;
+
+/**
+ * CompressorOutputStream for the LZ4 block format.
+ *
+ * @see <a href="http://lz4.github.io/lz4/lz4_Block_format.html">LZ4 Block Format Description</a>
+ * @since 1.14
+ */
+public class BlockLZ4CompressorOutputStream extends CompressorOutputStream {
+
+    /*
+
+      The LZ4 block format has a few properties that make it less
+      straight-forward than one would hope:
+
+      * literal blocks and back-references must come in pairs (except
+        for the very last literal block), so consecutive literal
+        blocks created by the compressor must be merged into a single
+        block.
+
+      * the start of a literal/back-reference pair contains the length
+        of the copy (at least some part of it) so we can't start
+        writing the literal before we know how long the next copy is
+        going to be.
+
+      * there is a special rule for the final blocks
+
+        > There are specific parsing rules to respect in order to remain
+        > compatible with assumptions made by the decoder :
+        >
+        >     1. The last 5 bytes are always literals
+        >
+        >     2. The last match must start at least 12 bytes before end of
+        >        block. Consequently, a block with less than 13 bytes cannot be
+        >        compressed.
+
+        which means any back-reference may need to get rewritten as a
+        literal block unless we know the next block is at least of
+        length 5 and the sum of this block's length and offset and the
+        next block's length is at least twelve.
+
+    */
+
+    private final LZ77Compressor compressor;
+    private final OutputStream os;
+    private final ByteUtils.ByteConsumer consumer;
+
+    // used in one-arg write method
+    private final byte[] oneByte = new byte[1];
+
+    private boolean finished = false;
+
+    /**
+     * Creates a new LZ4 output stream.
+     *
+     * @param os
+     *            An OutputStream to read compressed data from
+     *
+     * @throws IOException if reading fails
+     */
+    public BlockLZ4CompressorOutputStream(final OutputStream os) throws IOException {
+        this.os = os;
+        consumer = new ByteUtils.OutputStreamByteConsumer(os);
+        int maxLen = BlockLZ4CompressorInputStream.WINDOW_SIZE - 1;
+        compressor = new LZ77Compressor(new Parameters(BlockLZ4CompressorInputStream.WINDOW_SIZE, 4, maxLen, maxLen,
+            maxLen),
+            new LZ77Compressor.Callback() {
+                public void accept(LZ77Compressor.Block block) throws IOException {
+                    //System.err.println(block);
+                    if (block instanceof LZ77Compressor.LiteralBlock) {
+                        addLiteralBlock((LZ77Compressor.LiteralBlock) block);
+                    } else if (block instanceof LZ77Compressor.BackReference) {
+                        addBackReference((LZ77Compressor.BackReference) block);
+                    } else if (block instanceof LZ77Compressor.EOD) {
+                        writeFinalLiteralBlock();
+                    }
+                }
+            });
+    }
+
+    @Override
+    public void write(int b) throws IOException {
+        oneByte[0] = (byte) (b & 0xff);
+        write(oneByte);
+    }
+
+    @Override
+    public void write(byte[] data, int off, int len) throws IOException {
+        compressor.compress(data, off, len);
+    }
+
+    @Override
+    public void close() throws IOException {
+        finish();
+        os.close();
+    }
+
+    /**
+     * Compresses all remaining data and writes it to the stream,
+     * doesn't close the underlying stream.
+     * @throws IOException if an error occurs
+     */
+    public void finish() throws IOException {
+        if (!finished) {
+            compressor.finish();
+            finished = true;
+        }
+    }
+
+    private void addLiteralBlock(LZ77Compressor.LiteralBlock block) throws IOException {
+    }
+
+    private void addBackReference(LZ77Compressor.BackReference block) throws IOException {
+    }
+
+    private void writeFinalLiteralBlock() throws IOException {
+    }
+}