commons-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bode...@apache.org
Subject commons-compress git commit: COMPRESS-271 implement writing of block checksum
Date Fri, 27 Jan 2017 05:34:03 GMT
Repository: commons-compress
Updated Branches:
  refs/heads/master 68fa68928 -> 12b7b4d93


COMPRESS-271 implement writing of block checksum


Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/12b7b4d9
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/12b7b4d9
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/12b7b4d9

Branch: refs/heads/master
Commit: 12b7b4d93b945c77b801a26dab153d7f871aee49
Parents: 68fa689
Author: Stefan Bodewig <bodewig@apache.org>
Authored: Fri Jan 27 06:33:28 2017 +0100
Committer: Stefan Bodewig <bodewig@apache.org>
Committed: Fri Jan 27 06:33:28 2017 +0100

----------------------------------------------------------------------
 .../lz4/FramedLZ4CompressorOutputStream.java    | 33 +++++++++++++++-----
 .../lz4/FramedLZ4CompressorRoundtripTest.java   | 10 +++---
 2 files changed, 32 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/commons-compress/blob/12b7b4d9/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
index 525e05b..aa59424 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java
@@ -40,7 +40,6 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream
{
     /*
      * TODO before releasing 1.14:
      *
-     * + xxhash32 checksum creation for blocks
      * + block dependence
      */
 
@@ -55,8 +54,10 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream
{
     private boolean finished = false;
     private int currentIndex = 0;
 
-    // used for frame header checksum and content checksum, if present
+    // used for frame header checksum and content checksum, if requested
     private final XXHash32 contentHash = new XXHash32();
+    // used for block checksum, if requested
+    private final XXHash32 blockHash;
 
     /**
      * The block sizes supported by the format.
@@ -89,7 +90,7 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream
{
      */
     public static class Parameters {
         private final BlockSize blockSize;
-        private final boolean withContentChecksum;
+        private final boolean withContentChecksum, withBlockChecksum;
 
         /**
          * The default parameters of 4M block size, enabled content
@@ -97,20 +98,25 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream
{
          *
          * <p>This matches the defaults of the lz4 command line utility.</p>
          */
-        public static Parameters DEFAULT = new Parameters(BlockSize.M4, true);
+        public static Parameters DEFAULT = new Parameters(BlockSize.M4, true, false);
 
         /**
          * Sets up custom parameters for the LZ4 stream.
          * @param blockSize the size of a single block.
          * @param withContentChecksum whether to write a content checksum
+         * @param withBlockChecksum whether to write a block checksum.
+         * Note that block checksums are not supported by the lz4
+         * command line utility
          */
-        public Parameters(BlockSize blockSize, boolean withContentChecksum) {
+        public Parameters(BlockSize blockSize, boolean withContentChecksum, boolean withBlockChecksum)
{
             this.blockSize = blockSize;
             this.withContentChecksum = withContentChecksum;
+            this.withBlockChecksum = withBlockChecksum;
         }
         @Override
         public String toString() {
-            return "LZ4 Parameters with BlockSize " + blockSize + ", withContentChecksum
" + withContentChecksum;
+            return "LZ4 Parameters with BlockSize " + blockSize + ", withContentChecksum
" + withContentChecksum
+                + ", withBlockChecksum " + withBlockChecksum;
         }
     }
 
@@ -135,6 +141,7 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream
{
         this.params = params;
         blockData = new byte[params.blockSize.getSize()];
         this.out = out;
+        blockHash = params.withBlockChecksum ? new XXHash32() : null;
         out.write(FramedLZ4CompressorInputStream.LZ4_SIGNATURE);
         writeFrameDescriptor();
     }
@@ -189,6 +196,9 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream
{
         if (params.withContentChecksum) {
             flags |= FramedLZ4CompressorInputStream.CONTENT_CHECKSUM_MASK;
         }
+        if (params.withBlockChecksum) {
+            flags |= FramedLZ4CompressorInputStream.BLOCK_CHECKSUM_MASK;
+        }
         out.write(flags);
         contentHash.update(flags);
         int bd = params.blockSize.getIndex() << 4;
@@ -208,11 +218,20 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream
{
             ByteUtils.toLittleEndian(out, currentIndex | FramedLZ4CompressorInputStream.UNCOMPRESSED_FLAG_MASK,
                 4);
             out.write(blockData, 0, currentIndex);
+            if (params.withBlockChecksum) {
+                blockHash.update(blockData, 0, currentIndex);
+            }
         } else {
             ByteUtils.toLittleEndian(out, b.length, 4);
             out.write(b);
+            if (params.withBlockChecksum) {
+                blockHash.update(b, 0, b.length);
+            }
+        }
+        if (params.withBlockChecksum) {
+            ByteUtils.toLittleEndian(out, blockHash.getValue(), 4);
+            blockHash.reset();
         }
-        // TODO block checksum
         currentIndex = 0;
     }
 

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/12b7b4d9/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java
b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java
index a6aa5aa..1a14352 100644
--- a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java
+++ b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java
@@ -39,12 +39,14 @@ public final class FramedLZ4CompressorRoundtripTest extends AbstractTestCase
{
     @Parameters(name = "using {0}")
     public static Collection<Object[]> factory() {
         return Arrays.asList(new Object[][] {
-            new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.K64,
true) },
-            new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.K256,
true) },
-            new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.M1,
true) },
+             new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.K64,
true, false) },
+            new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.K256,
true, false) },
+            new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.M1,
true, false) },
             new Object[] { FramedLZ4CompressorOutputStream.Parameters.DEFAULT },
             // default without content checksum
-            new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.M4,
false) },
+            new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.M4,
false, false) },
+            // default with block checksum
+            new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.M4,
true, true) },
         });
     }
 


Mime
View raw message