commons-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bode...@apache.org
Subject svn commit: r1548947 - in /commons/proper/compress/trunk/src: main/java/org/apache/commons/compress/compressors/snappy/ main/java/org/apache/commons/compress/utils/ test/java/org/apache/commons/compress/compressors/ test/java/org/apache/commons/compres...
Date Sat, 07 Dec 2013 18:01:01 GMT
Author: bodewig
Date: Sat Dec  7 18:01:01 2013
New Revision: 1548947

URL: http://svn.apache.org/r1548947
Log:
COMPRESS-147 checksum verification

Added:
    commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/ChecksumVerifyingInputStream.java
      - copied, changed from r1548937, commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java
    commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/PureJavaCrc32C.java
  (contents, props changed)
      - copied, changed from r1548942, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java
    commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/snappy/
    commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStreamTest.java
      - copied, changed from r1548932, commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java
Modified:
    commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java
    commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java
    commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java
    commons/proper/compress/trunk/src/test/resources/mixed.txt.sz

Modified: commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java
URL: http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java?rev=1548947&r1=1548946&r2=1548947&view=diff
==============================================================================
--- commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java
(original)
+++ commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java
Sat Dec  7 18:01:01 2013
@@ -26,6 +26,7 @@ import java.util.Arrays;
 import org.apache.commons.compress.compressors.CompressorInputStream;
 import org.apache.commons.compress.utils.BoundedInputStream;
 import org.apache.commons.compress.utils.IOUtils;
+import org.apache.commons.compress.utils.PureJavaCrc32C;
 
 /**
  * CompressorInputStream for the framing Snappy format.
@@ -36,6 +37,11 @@ import org.apache.commons.compress.utils
  * @since 1.7
  */
 public class FramedSnappyCompressorInputStream extends CompressorInputStream {
+    /**
+     * package private for tests only.
+     */
+    static final long MASK_OFFSET = 0xa282ead8L;
+
     private static final int STREAM_IDENTIFIER_TYPE = 0xff;
     private static final int COMPRESSED_CHUNK_TYPE = 0;
     private static final int UNCOMPRESSED_CHUNK_TYPE = 1;
@@ -61,6 +67,8 @@ public class FramedSnappyCompressorInput
     private boolean endReached, inUncompressedChunk;
 
     private int uncompressedBytesRemaining;
+    private long expectedChecksum = -1;
+    private PureJavaCrc32C checksum = new PureJavaCrc32C();
 
     /**
      * Constructs a new input stream that decompresses snappy-framed-compressed data
@@ -143,10 +151,14 @@ public class FramedSnappyCompressorInput
                 count(currentCompressedChunk.getBytesRead() - before);
             }
         }
+        if (read > 0) {
+            checksum.update(b, off, read);
+        }
         return read;
     }
 
     private void readNextBlock() throws IOException {
+        verifyLastChecksumAndReset();
         inUncompressedChunk = false;
         int type = readOneByte();
         if (type == -1) {
@@ -167,10 +179,10 @@ public class FramedSnappyCompressorInput
         } else if (type == UNCOMPRESSED_CHUNK_TYPE) {
             inUncompressedChunk = true;
             uncompressedBytesRemaining = readSize() - 4 /* CRC */;
-            readCrc();
+            expectedChecksum = unmask(readCrc());
         } else if (type == COMPRESSED_CHUNK_TYPE) {
-            int size = readSize() - 4 /* CRC */;
-            readCrc();
+            long size = readSize() - 4 /* CRC */;
+            expectedChecksum = unmask(readCrc());
             currentCompressedChunk =
                 new SnappyCompressorInputStream(new BoundedInputStream(in, size));
             // constructor reads uncompressed size
@@ -182,13 +194,26 @@ public class FramedSnappyCompressorInput
         }
     }
 
-    private void readCrc() throws IOException {
+    private long readCrc() throws IOException {
         byte[] b = new byte[4];
         int read = IOUtils.readFully(in, b);
         count(read);
         if (read != 4) {
             throw new IOException("premature end of stream");
         }
+        long crc = 0;
+        for (int i = 0; i < 4; i++) {
+            crc |= (b[i] & 0xFFL) << (8 * i);
+        }
+        return crc;
+    }
+
+    static long unmask(long x) {
+        // ugly, maybe we should just have used ints and deal with the
+        // overflow
+        x -= MASK_OFFSET;
+        x &= 0xffffFFFFL;
+        return ((x >> 17) | (x << 15)) & 0xffffFFFFL;
     }
 
     private int readSize() throws IOException {
@@ -231,6 +256,14 @@ public class FramedSnappyCompressorInput
         return -1;
     }
 
+    private void verifyLastChecksumAndReset() throws IOException {
+        if (expectedChecksum >= 0 && expectedChecksum != checksum.getValue())
{
+            throw new IOException("Checksum verification failed");
+        }
+        expectedChecksum = -1;
+        checksum.reset();
+    }
+
     /**
      * Checks if the signature matches what is expected for a .sz file.
      *

Modified: commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java
URL: http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java?rev=1548947&r1=1548946&r2=1548947&view=diff
==============================================================================
--- commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java
(original)
+++ commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java
Sat Dec  7 18:01:01 2013
@@ -17,7 +17,6 @@
  */
 package org.apache.commons.compress.utils;
 
-import java.io.IOException;
 import java.io.InputStream;
 import java.util.zip.CRC32;
 
@@ -27,82 +26,10 @@ import java.util.zip.CRC32;
  * @NotThreadSafe
  * @since 1.6
  */
-public class CRC32VerifyingInputStream extends InputStream {
-    private final InputStream in;
-    private long bytesRemaining;
-    private final long expectedCrc32;
-    private final CRC32 crc32 = new CRC32();
+public class CRC32VerifyingInputStream extends ChecksumVerifyingInputStream {
     
     public CRC32VerifyingInputStream(final InputStream in, final long size, final long expectedCrc32)
{
-        this.in = in;
-        this.expectedCrc32 = expectedCrc32;
-        this.bytesRemaining = size;
+        super(new CRC32(), in, size, expectedCrc32);
     }
 
-    /**
-     * Reads a single byte from the stream
-     * @throws IOException if the underlying stream throws or the
-     * stream is exhausted and the CRC doesn't match the expected
-     * value
-     */
-    @Override
-    public int read() throws IOException {
-        if (bytesRemaining <= 0) {
-            return -1;
-        }
-        int ret = in.read();
-        if (ret >= 0) {
-            crc32.update(ret);
-            --bytesRemaining;
-        }
-        if (bytesRemaining == 0 && expectedCrc32 != crc32.getValue()) {
-            throw new IOException("CRC32 verification failed");
-        }
-        return ret;
-    }
-
-    /**
-     * Reads a byte array from the stream
-     * @throws IOException if the underlying stream throws or the
-     * stream is exhausted and the CRC doesn't match the expected
-     * value
-     */
-    @Override
-    public int read(byte[] b) throws IOException {
-        return read(b, 0, b.length);
-    }
-
-    /**
-     * Reads from the stream into a byte array.
-     * @throws IOException if the underlying stream throws or the
-     * stream is exhausted and the CRC doesn't match the expected
-     * value
-     */
-    @Override
-    public int read(byte[] b, int off, int len) throws IOException {
-        int ret = in.read(b, off, len);
-        if (ret >= 0) {
-            crc32.update(b, off, ret);
-            bytesRemaining -= ret;
-        }
-        if (bytesRemaining <= 0 && expectedCrc32 != crc32.getValue()) {
-            throw new IOException("CRC32 verification failed");
-        }
-        return ret;
-    }
-
-    @Override
-    public long skip(long n) throws IOException {
-        // Can't really skip, we have to hash everything to verify the checksum
-        if (read() >= 0) {
-            return 1;
-        } else {
-            return 0;
-        }
-    }
-
-    @Override
-    public void close() throws IOException {
-        in.close();
-    }
 }

Copied: commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/ChecksumVerifyingInputStream.java
(from r1548937, commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java)
URL: http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/ChecksumVerifyingInputStream.java?p2=commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/ChecksumVerifyingInputStream.java&p1=commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java&r1=1548937&r2=1548947&rev=1548947&view=diff
==============================================================================
--- commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java
(original)
+++ commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/ChecksumVerifyingInputStream.java
Sat Dec  7 18:01:01 2013
@@ -19,30 +19,32 @@ package org.apache.commons.compress.util
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.zip.CRC32;
+import java.util.zip.Checksum;
 
 /**
- * A stream that verifies the CRC of the data read once the stream is
+ * A stream that verifies the checksum of the data read once the stream is
  * exhausted.
  * @NotThreadSafe
- * @since 1.6
+ * @since 1.7
  */
-public class CRC32VerifyingInputStream extends InputStream {
+public class ChecksumVerifyingInputStream extends InputStream {
     private final InputStream in;
     private long bytesRemaining;
-    private final long expectedCrc32;
-    private final CRC32 crc32 = new CRC32();
+    private final long expectedChecksum;
+    private final Checksum checksum;
     
-    public CRC32VerifyingInputStream(final InputStream in, final long size, final long expectedCrc32)
{
+    public ChecksumVerifyingInputStream(final Checksum checksum, final InputStream in,
+                                        final long size, final long expectedChecksum) {
+        this.checksum = checksum;
         this.in = in;
-        this.expectedCrc32 = expectedCrc32;
+        this.expectedChecksum = expectedChecksum;
         this.bytesRemaining = size;
     }
 
     /**
      * Reads a single byte from the stream
      * @throws IOException if the underlying stream throws or the
-     * stream is exhausted and the CRC doesn't match the expected
+     * stream is exhausted and the Checksum doesn't match the expected
      * value
      */
     @Override
@@ -52,11 +54,11 @@ public class CRC32VerifyingInputStream e
         }
         int ret = in.read();
         if (ret >= 0) {
-            crc32.update(ret);
+            checksum.update(ret);
             --bytesRemaining;
         }
-        if (bytesRemaining == 0 && expectedCrc32 != crc32.getValue()) {
-            throw new IOException("CRC32 verification failed");
+        if (bytesRemaining == 0 && expectedChecksum != checksum.getValue()) {
+            throw new IOException("Checksum verification failed");
         }
         return ret;
     }
@@ -64,7 +66,7 @@ public class CRC32VerifyingInputStream e
     /**
      * Reads a byte array from the stream
      * @throws IOException if the underlying stream throws or the
-     * stream is exhausted and the CRC doesn't match the expected
+     * stream is exhausted and the Checksum doesn't match the expected
      * value
      */
     @Override
@@ -75,18 +77,18 @@ public class CRC32VerifyingInputStream e
     /**
      * Reads from the stream into a byte array.
      * @throws IOException if the underlying stream throws or the
-     * stream is exhausted and the CRC doesn't match the expected
+     * stream is exhausted and the Checksum doesn't match the expected
      * value
      */
     @Override
     public int read(byte[] b, int off, int len) throws IOException {
         int ret = in.read(b, off, len);
         if (ret >= 0) {
-            crc32.update(b, off, ret);
+            checksum.update(b, off, ret);
             bytesRemaining -= ret;
         }
-        if (bytesRemaining <= 0 && expectedCrc32 != crc32.getValue()) {
-            throw new IOException("CRC32 verification failed");
+        if (bytesRemaining <= 0 && expectedChecksum != checksum.getValue()) {
+            throw new IOException("Checksum verification failed");
         }
         return ret;
     }

Copied: commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/PureJavaCrc32C.java
(from r1548942, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java)
URL: http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/PureJavaCrc32C.java?p2=commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/PureJavaCrc32C.java&p1=hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java&r1=1548942&r2=1548947&rev=1548947&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java
(original)
+++ commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/PureJavaCrc32C.java
Sat Dec  7 18:01:01 2013
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -18,20 +18,20 @@
  * Some portions of this file Copyright (c) 2004-2006 Intel Corportation
  * and licensed under the BSD license.
  */
-package org.apache.hadoop.util;
+package org.apache.commons.compress.utils;
 
 import java.util.zip.Checksum;
 
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
 /**
  * A pure-java implementation of the CRC32 checksum that uses
  * the CRC32-C polynomial, the same polynomial used by iSCSI
  * and implemented on many Intel chipsets supporting SSE4.2.
+ *
+ * <p>This file is a copy of the implementation at the Apache Hadoop project.</p>
+ * @see "http://svn.apache.org/repos/asf/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java"
+ * @NotThreadSafe
+ * @since 1.7
  */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
 public class PureJavaCrc32C implements Checksum {
 
   /** the current CRC value, bit-flipped */

Propchange: commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/PureJavaCrc32C.java
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Sat Dec  7 18:01:01 2013
@@ -0,0 +1,9 @@
+/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java:1152502-1296519
+/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java:1360400-1480829
+/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java:1306184-1342109
+/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java:1363593-1396941
+/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java:1430995-1467533
+/hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java:1509426-1536569
+/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java:1526848,1541667
+/hadoop/core/branches/branch-0.19/core/src/java/org/apache/hadoop/util/PureJavaCrc32C.java:713112
+/hadoop/core/trunk/src/core/org/apache/hadoop/util/PureJavaCrc32C.java:776175-785643,785929-786278

Modified: commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java
URL: http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java?rev=1548947&r1=1548946&r2=1548947&view=diff
==============================================================================
--- commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java
(original)
+++ commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java
Sat Dec  7 18:01:01 2013
@@ -21,38 +21,19 @@ package org.apache.commons.compress.comp
 import static org.junit.Assert.assertArrayEquals;
 
 import java.io.BufferedInputStream;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.Arrays;
 
 import org.apache.commons.compress.AbstractTestCase;
-import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
 import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorInputStream;
 import org.apache.commons.compress.utils.IOUtils;
 
 public final class FramedSnappyTestCase
     extends AbstractTestCase {
 
-    public void testMatches() throws IOException {
-        assertFalse(FramedSnappyCompressorInputStream.matches(new byte[10], 10));
-        byte[] b = new byte[12];
-        final File input = getFile("bla.tar.sz");
-        FileInputStream in = new FileInputStream(input);
-        try {
-            IOUtils.readFully(in, b);
-        } finally {
-            in.close();
-        }
-        assertFalse(FramedSnappyCompressorInputStream.matches(b, 9));
-        assertTrue(FramedSnappyCompressorInputStream.matches(b, 10));
-        assertTrue(FramedSnappyCompressorInputStream.matches(b, 12));
-    }
-
     public void testDefaultExtraction() throws Exception {
         testUnarchive(new StreamWrapper<CompressorInputStream>() {
             public CompressorInputStream wrap(InputStream is) throws IOException {
@@ -79,114 +60,6 @@ public final class FramedSnappyTestCase
         });
     }
 
-    /**
-     * Something big enough to make buffers slide.
-     */
-    public void testLoremIpsum() throws Exception {
-        final FileInputStream isSz = new FileInputStream(getFile("lorem-ipsum.txt.sz"));
-        final File outputSz = new File(dir, "lorem-ipsum.1");
-        final File outputGz = new File(dir, "lorem-ipsum.2");
-        try {
-            CompressorInputStream in = new FramedSnappyCompressorInputStream(isSz);
-            FileOutputStream out = null;
-            try {
-                out = new FileOutputStream(outputSz);
-                IOUtils.copy(in, out);
-            } finally {
-                if (out != null) {
-                    out.close();
-                }
-                in.close();
-            }
-            final FileInputStream isGz = new FileInputStream(getFile("lorem-ipsum.txt.gz"));
-            try {
-                in = new GzipCompressorInputStream(isGz);
-                try {
-                    out = new FileOutputStream(outputGz);
-                    IOUtils.copy(in, out);
-                } finally {
-                    if (out != null) {
-                        out.close();
-                    }
-                    in.close();
-                }
-            } finally {
-                isGz.close();
-            }
-        } finally {
-            isSz.close();
-        }
-
-        final FileInputStream sz = new FileInputStream(outputSz);
-        try {
-            FileInputStream gz = new FileInputStream(outputGz);
-            try {
-                assertArrayEquals(IOUtils.toByteArray(sz),
-                                  IOUtils.toByteArray(gz));
-            } finally {
-                gz.close();
-            }
-        } finally {
-            sz.close();
-        }
-    }
-
-    public void testRemainingChunkTypes() throws Exception {
-        final FileInputStream isSz = new FileInputStream(getFile("mixed.txt.sz"));
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        try {
-            CompressorInputStream in = new FramedSnappyCompressorInputStream(isSz);
-            IOUtils.copy(in, out);
-            out.close();
-        } finally {
-            isSz.close();
-        }
-
-        assertArrayEquals(new byte[] { '1', '2', '3', '4',
-                                       '5', '6', '7', '8', '9',
-                                       '5', '6', '7', '8', '9',
-                                       '5', '6', '7', '8', '9',
-                                       '5', '6', '7', '8', '9',
-                                       '5', '6', '7', '8', '9', 10,
-                                       '1', '2', '3', '4',
-                                       '1', '2', '3', '4',
-            }, out.toByteArray());
-    }
-
-    public void testAvailable() throws Exception {
-        final FileInputStream isSz = new FileInputStream(getFile("mixed.txt.sz"));
-        try {
-            CompressorInputStream in = new FramedSnappyCompressorInputStream(isSz);
-            assertEquals(0, in.available()); // no chunk read so far
-            assertEquals('1', in.read());
-            assertEquals(3, in.available()); // remainder of first uncompressed block
-            assertEquals(3, in.read(new byte[5], 0, 3));
-            assertEquals('5', in.read());
-            assertEquals(4, in.available()); // remainder of literal
-            assertEquals(4, in.read(new byte[5], 0, 4));
-            assertEquals('5', in.read());
-            assertEquals(19, in.available()); // remainder of copy
-            in.close();
-        } finally {
-            isSz.close();
-        }
-    }
-
-    public void testUnskippableChunk() {
-        byte[] input = new byte[] {
-            (byte) 0xff, 6, 0, 0, 's', 'N', 'a', 'P', 'p', 'Y',
-            2, 2, 0, 0, 1, 1
-        };
-        try {
-            CompressorInputStream in =
-                new FramedSnappyCompressorInputStream(new ByteArrayInputStream(input));
-            in.read();
-            fail("expected an exception");
-        } catch (IOException ex) {
-            assertTrue(ex.getMessage().indexOf("unskippable chunk") > -1);
-        }
-    }
-
     private void testUnarchive(StreamWrapper<CompressorInputStream> wrapper) throws
Exception {
         final File input = getFile("bla.tar.sz");
         final File output = new File(dir, "bla.tar");
@@ -223,4 +96,5 @@ public final class FramedSnappyTestCase
             written.close();
         }
     }
+
 }

Copied: commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStreamTest.java
(from r1548932, commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java)
URL: http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStreamTest.java?p2=commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStreamTest.java&p1=commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java&r1=1548932&r2=1548947&rev=1548947&view=diff
==============================================================================
--- commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java
(original)
+++ commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStreamTest.java
Sat Dec  7 18:01:01 2013
@@ -16,11 +16,10 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.commons.compress.compressors;
+package org.apache.commons.compress.compressors.snappy;
 
 import static org.junit.Assert.assertArrayEquals;
 
-import java.io.BufferedInputStream;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
@@ -32,10 +31,9 @@ import java.util.Arrays;
 
 import org.apache.commons.compress.AbstractTestCase;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
-import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorInputStream;
 import org.apache.commons.compress.utils.IOUtils;
 
-public final class FramedSnappyTestCase
+public final class FramedSnappyCompressorInputStreamTest
     extends AbstractTestCase {
 
     public void testMatches() throws IOException {
@@ -53,32 +51,6 @@ public final class FramedSnappyTestCase
         assertTrue(FramedSnappyCompressorInputStream.matches(b, 12));
     }
 
-    public void testDefaultExtraction() throws Exception {
-        testUnarchive(new StreamWrapper<CompressorInputStream>() {
-            public CompressorInputStream wrap(InputStream is) throws IOException {
-                return new FramedSnappyCompressorInputStream(is);
-            }
-        });
-    }
-
-    public void testDefaultExtractionViaFactory() throws Exception {
-        testUnarchive(new StreamWrapper<CompressorInputStream>() {
-            public CompressorInputStream wrap(InputStream is) throws Exception {
-                return new CompressorStreamFactory()
-                    .createCompressorInputStream(CompressorStreamFactory.SNAPPY_FRAMED,
-                                                 is);
-            }
-        });
-    }
-
-    public void testDefaultExtractionViaFactoryAutodetection() throws Exception {
-        testUnarchive(new StreamWrapper<CompressorInputStream>() {
-            public CompressorInputStream wrap(InputStream is) throws Exception {
-                return new CompressorStreamFactory().createCompressorInputStream(is);
-            }
-        });
-    }
-
     /**
      * Something big enough to make buffers slide.
      */
@@ -87,7 +59,7 @@ public final class FramedSnappyTestCase
         final File outputSz = new File(dir, "lorem-ipsum.1");
         final File outputGz = new File(dir, "lorem-ipsum.2");
         try {
-            CompressorInputStream in = new FramedSnappyCompressorInputStream(isSz);
+            InputStream in = new FramedSnappyCompressorInputStream(isSz);
             FileOutputStream out = null;
             try {
                 out = new FileOutputStream(outputSz);
@@ -135,7 +107,7 @@ public final class FramedSnappyTestCase
         final FileInputStream isSz = new FileInputStream(getFile("mixed.txt.sz"));
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         try {
-            CompressorInputStream in = new FramedSnappyCompressorInputStream(isSz);
+            FramedSnappyCompressorInputStream in = new FramedSnappyCompressorInputStream(isSz);
             IOUtils.copy(in, out);
             out.close();
         } finally {
@@ -156,7 +128,7 @@ public final class FramedSnappyTestCase
     public void testAvailable() throws Exception {
         final FileInputStream isSz = new FileInputStream(getFile("mixed.txt.sz"));
         try {
-            CompressorInputStream in = new FramedSnappyCompressorInputStream(isSz);
+            FramedSnappyCompressorInputStream in = new FramedSnappyCompressorInputStream(isSz);
             assertEquals(0, in.available()); // no chunk read so far
             assertEquals('1', in.read());
             assertEquals(3, in.available()); // remainder of first uncompressed block
@@ -178,7 +150,7 @@ public final class FramedSnappyTestCase
             2, 2, 0, 0, 1, 1
         };
         try {
-            CompressorInputStream in =
+            FramedSnappyCompressorInputStream in =
                 new FramedSnappyCompressorInputStream(new ByteArrayInputStream(input));
             in.read();
             fail("expected an exception");
@@ -187,40 +159,21 @@ public final class FramedSnappyTestCase
         }
     }
 
-    private void testUnarchive(StreamWrapper<CompressorInputStream> wrapper) throws
Exception {
-        final File input = getFile("bla.tar.sz");
-        final File output = new File(dir, "bla.tar");
-        final FileInputStream is = new FileInputStream(input);
-        try {
-            // the intermediate BufferedInputStream is there for mark
-            // support in the autodetection test
-            final CompressorInputStream in = wrapper.wrap(new BufferedInputStream(is));
-            FileOutputStream out = null;
-            try {
-                out = new FileOutputStream(output);
-                IOUtils.copy(in, out);
-                assertEquals(995, in.getBytesRead());
-            } finally {
-                if (out != null) {
-                    out.close();
-                }
-                in.close();
-            }
-        } finally {
-            is.close();
-        }
-        final File original = getFile("bla.tar");
-        final FileInputStream written = new FileInputStream(output);
-        try {
-            FileInputStream orig = new FileInputStream(original);
-            try {
-                assertArrayEquals(IOUtils.toByteArray(written),
-                                  IOUtils.toByteArray(orig));
-            } finally {
-                orig.close();
-            }
-        } finally {
-            written.close();
-        }
+    public void testChecksumUnmasking() {
+        testChecksumUnmasking(0xc757l);
+        testChecksumUnmasking(0xffffc757l);
+    }
+
+    public void testChecksumUnmasking(long x) {
+        assertEquals(Long.toHexString(x),
+                     Long.toHexString(FramedSnappyCompressorInputStream
+                                      .unmask(mask(x))));
+    }
+
+    private long mask(long x) {
+        return (((x >>> 15) | (x << 17))
+                + FramedSnappyCompressorInputStream.MASK_OFFSET)
+             & 0xffffFFFFL;
     }
+
 }

Modified: commons/proper/compress/trunk/src/test/resources/mixed.txt.sz
URL: http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/test/resources/mixed.txt.sz?rev=1548947&r1=1548946&r2=1548947&view=diff
==============================================================================
Binary files - no diff available.



Mime
View raw message