Return-Path: X-Original-To: apmail-commons-commits-archive@minotaur.apache.org Delivered-To: apmail-commons-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 129AB109C5 for ; Sat, 7 Dec 2013 18:01:30 +0000 (UTC) Received: (qmail 48703 invoked by uid 500); 7 Dec 2013 18:01:29 -0000 Delivered-To: apmail-commons-commits-archive@commons.apache.org Received: (qmail 48640 invoked by uid 500); 7 Dec 2013 18:01:29 -0000 Mailing-List: contact commits-help@commons.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@commons.apache.org Delivered-To: mailing list commits@commons.apache.org Received: (qmail 48633 invoked by uid 99); 7 Dec 2013 18:01:29 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 07 Dec 2013 18:01:29 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 07 Dec 2013 18:01:24 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id E246C2388A5B; Sat, 7 Dec 2013 18:01:01 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1548947 - in /commons/proper/compress/trunk/src: main/java/org/apache/commons/compress/compressors/snappy/ main/java/org/apache/commons/compress/utils/ test/java/org/apache/commons/compress/compressors/ test/java/org/apache/commons/compres... Date: Sat, 07 Dec 2013 18:01:01 -0000 To: commits@commons.apache.org From: bodewig@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20131207180101.E246C2388A5B@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: bodewig Date: Sat Dec 7 18:01:01 2013 New Revision: 1548947 URL: http://svn.apache.org/r1548947 Log: COMPRESS-147 checksum verification Added: commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/ChecksumVerifyingInputStream.java - copied, changed from r1548937, commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/PureJavaCrc32C.java (contents, props changed) - copied, changed from r1548942, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/snappy/ commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStreamTest.java - copied, changed from r1548932, commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java Modified: commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java commons/proper/compress/trunk/src/test/resources/mixed.txt.sz Modified: commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java URL: http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java?rev=1548947&r1=1548946&r2=1548947&view=diff ============================================================================== --- commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java (original) +++ commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStream.java Sat Dec 7 18:01:01 2013 @@ -26,6 +26,7 @@ import java.util.Arrays; import org.apache.commons.compress.compressors.CompressorInputStream; import org.apache.commons.compress.utils.BoundedInputStream; import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.compress.utils.PureJavaCrc32C; /** * CompressorInputStream for the framing Snappy format. @@ -36,6 +37,11 @@ import org.apache.commons.compress.utils * @since 1.7 */ public class FramedSnappyCompressorInputStream extends CompressorInputStream { + /** + * package private for tests only. + */ + static final long MASK_OFFSET = 0xa282ead8L; + private static final int STREAM_IDENTIFIER_TYPE = 0xff; private static final int COMPRESSED_CHUNK_TYPE = 0; private static final int UNCOMPRESSED_CHUNK_TYPE = 1; @@ -61,6 +67,8 @@ public class FramedSnappyCompressorInput private boolean endReached, inUncompressedChunk; private int uncompressedBytesRemaining; + private long expectedChecksum = -1; + private PureJavaCrc32C checksum = new PureJavaCrc32C(); /** * Constructs a new input stream that decompresses snappy-framed-compressed data @@ -143,10 +151,14 @@ public class FramedSnappyCompressorInput count(currentCompressedChunk.getBytesRead() - before); } } + if (read > 0) { + checksum.update(b, off, read); + } return read; } private void readNextBlock() throws IOException { + verifyLastChecksumAndReset(); inUncompressedChunk = false; int type = readOneByte(); if (type == -1) { @@ -167,10 +179,10 @@ public class FramedSnappyCompressorInput } else if (type == UNCOMPRESSED_CHUNK_TYPE) { inUncompressedChunk = true; uncompressedBytesRemaining = readSize() - 4 /* CRC */; - readCrc(); + expectedChecksum = unmask(readCrc()); } else if (type == COMPRESSED_CHUNK_TYPE) { - int size = readSize() - 4 /* CRC */; - readCrc(); + long size = readSize() - 4 /* CRC */; + expectedChecksum = unmask(readCrc()); currentCompressedChunk = new SnappyCompressorInputStream(new BoundedInputStream(in, size)); // constructor reads uncompressed size @@ -182,13 +194,26 @@ public class FramedSnappyCompressorInput } } - private void readCrc() throws IOException { + private long readCrc() throws IOException { byte[] b = new byte[4]; int read = IOUtils.readFully(in, b); count(read); if (read != 4) { throw new IOException("premature end of stream"); } + long crc = 0; + for (int i = 0; i < 4; i++) { + crc |= (b[i] & 0xFFL) << (8 * i); + } + return crc; + } + + static long unmask(long x) { + // ugly, maybe we should just have used ints and deal with the + // overflow + x -= MASK_OFFSET; + x &= 0xffffFFFFL; + return ((x >> 17) | (x << 15)) & 0xffffFFFFL; } private int readSize() throws IOException { @@ -231,6 +256,14 @@ public class FramedSnappyCompressorInput return -1; } + private void verifyLastChecksumAndReset() throws IOException { + if (expectedChecksum >= 0 && expectedChecksum != checksum.getValue()) { + throw new IOException("Checksum verification failed"); + } + expectedChecksum = -1; + checksum.reset(); + } + /** * Checks if the signature matches what is expected for a .sz file. * Modified: commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java URL: http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java?rev=1548947&r1=1548946&r2=1548947&view=diff ============================================================================== --- commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java (original) +++ commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java Sat Dec 7 18:01:01 2013 @@ -17,7 +17,6 @@ */ package org.apache.commons.compress.utils; -import java.io.IOException; import java.io.InputStream; import java.util.zip.CRC32; @@ -27,82 +26,10 @@ import java.util.zip.CRC32; * @NotThreadSafe * @since 1.6 */ -public class CRC32VerifyingInputStream extends InputStream { - private final InputStream in; - private long bytesRemaining; - private final long expectedCrc32; - private final CRC32 crc32 = new CRC32(); +public class CRC32VerifyingInputStream extends ChecksumVerifyingInputStream { public CRC32VerifyingInputStream(final InputStream in, final long size, final long expectedCrc32) { - this.in = in; - this.expectedCrc32 = expectedCrc32; - this.bytesRemaining = size; + super(new CRC32(), in, size, expectedCrc32); } - /** - * Reads a single byte from the stream - * @throws IOException if the underlying stream throws or the - * stream is exhausted and the CRC doesn't match the expected - * value - */ - @Override - public int read() throws IOException { - if (bytesRemaining <= 0) { - return -1; - } - int ret = in.read(); - if (ret >= 0) { - crc32.update(ret); - --bytesRemaining; - } - if (bytesRemaining == 0 && expectedCrc32 != crc32.getValue()) { - throw new IOException("CRC32 verification failed"); - } - return ret; - } - - /** - * Reads a byte array from the stream - * @throws IOException if the underlying stream throws or the - * stream is exhausted and the CRC doesn't match the expected - * value - */ - @Override - public int read(byte[] b) throws IOException { - return read(b, 0, b.length); - } - - /** - * Reads from the stream into a byte array. - * @throws IOException if the underlying stream throws or the - * stream is exhausted and the CRC doesn't match the expected - * value - */ - @Override - public int read(byte[] b, int off, int len) throws IOException { - int ret = in.read(b, off, len); - if (ret >= 0) { - crc32.update(b, off, ret); - bytesRemaining -= ret; - } - if (bytesRemaining <= 0 && expectedCrc32 != crc32.getValue()) { - throw new IOException("CRC32 verification failed"); - } - return ret; - } - - @Override - public long skip(long n) throws IOException { - // Can't really skip, we have to hash everything to verify the checksum - if (read() >= 0) { - return 1; - } else { - return 0; - } - } - - @Override - public void close() throws IOException { - in.close(); - } } Copied: commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/ChecksumVerifyingInputStream.java (from r1548937, commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java) URL: http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/ChecksumVerifyingInputStream.java?p2=commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/ChecksumVerifyingInputStream.java&p1=commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java&r1=1548937&r2=1548947&rev=1548947&view=diff ============================================================================== --- commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/CRC32VerifyingInputStream.java (original) +++ commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/ChecksumVerifyingInputStream.java Sat Dec 7 18:01:01 2013 @@ -19,30 +19,32 @@ package org.apache.commons.compress.util import java.io.IOException; import java.io.InputStream; -import java.util.zip.CRC32; +import java.util.zip.Checksum; /** - * A stream that verifies the CRC of the data read once the stream is + * A stream that verifies the checksum of the data read once the stream is * exhausted. * @NotThreadSafe - * @since 1.6 + * @since 1.7 */ -public class CRC32VerifyingInputStream extends InputStream { +public class ChecksumVerifyingInputStream extends InputStream { private final InputStream in; private long bytesRemaining; - private final long expectedCrc32; - private final CRC32 crc32 = new CRC32(); + private final long expectedChecksum; + private final Checksum checksum; - public CRC32VerifyingInputStream(final InputStream in, final long size, final long expectedCrc32) { + public ChecksumVerifyingInputStream(final Checksum checksum, final InputStream in, + final long size, final long expectedChecksum) { + this.checksum = checksum; this.in = in; - this.expectedCrc32 = expectedCrc32; + this.expectedChecksum = expectedChecksum; this.bytesRemaining = size; } /** * Reads a single byte from the stream * @throws IOException if the underlying stream throws or the - * stream is exhausted and the CRC doesn't match the expected + * stream is exhausted and the Checksum doesn't match the expected * value */ @Override @@ -52,11 +54,11 @@ public class CRC32VerifyingInputStream e } int ret = in.read(); if (ret >= 0) { - crc32.update(ret); + checksum.update(ret); --bytesRemaining; } - if (bytesRemaining == 0 && expectedCrc32 != crc32.getValue()) { - throw new IOException("CRC32 verification failed"); + if (bytesRemaining == 0 && expectedChecksum != checksum.getValue()) { + throw new IOException("Checksum verification failed"); } return ret; } @@ -64,7 +66,7 @@ public class CRC32VerifyingInputStream e /** * Reads a byte array from the stream * @throws IOException if the underlying stream throws or the - * stream is exhausted and the CRC doesn't match the expected + * stream is exhausted and the Checksum doesn't match the expected * value */ @Override @@ -75,18 +77,18 @@ public class CRC32VerifyingInputStream e /** * Reads from the stream into a byte array. * @throws IOException if the underlying stream throws or the - * stream is exhausted and the CRC doesn't match the expected + * stream is exhausted and the Checksum doesn't match the expected * value */ @Override public int read(byte[] b, int off, int len) throws IOException { int ret = in.read(b, off, len); if (ret >= 0) { - crc32.update(b, off, ret); + checksum.update(b, off, ret); bytesRemaining -= ret; } - if (bytesRemaining <= 0 && expectedCrc32 != crc32.getValue()) { - throw new IOException("CRC32 verification failed"); + if (bytesRemaining <= 0 && expectedChecksum != checksum.getValue()) { + throw new IOException("Checksum verification failed"); } return ret; } Copied: commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/PureJavaCrc32C.java (from r1548942, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java) URL: http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/PureJavaCrc32C.java?p2=commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/PureJavaCrc32C.java&p1=hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java&r1=1548942&r2=1548947&rev=1548947&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java (original) +++ commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/PureJavaCrc32C.java Sat Dec 7 18:01:01 2013 @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,20 +18,20 @@ * Some portions of this file Copyright (c) 2004-2006 Intel Corportation * and licensed under the BSD license. */ -package org.apache.hadoop.util; +package org.apache.commons.compress.utils; import java.util.zip.Checksum; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - /** * A pure-java implementation of the CRC32 checksum that uses * the CRC32-C polynomial, the same polynomial used by iSCSI * and implemented on many Intel chipsets supporting SSE4.2. + * + *

This file is a copy of the implementation at the Apache Hadoop project.

+ * @see "http://svn.apache.org/repos/asf/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java" + * @NotThreadSafe + * @since 1.7 */ -@InterfaceAudience.Public -@InterfaceStability.Stable public class PureJavaCrc32C implements Checksum { /** the current CRC value, bit-flipped */ Propchange: commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/utils/PureJavaCrc32C.java ------------------------------------------------------------------------------ --- svn:mergeinfo (added) +++ svn:mergeinfo Sat Dec 7 18:01:01 2013 @@ -0,0 +1,9 @@ +/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java:1152502-1296519 +/hadoop/common/branches/HDFS-2802/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java:1360400-1480829 +/hadoop/common/branches/HDFS-3042/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java:1306184-1342109 +/hadoop/common/branches/HDFS-3077/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java:1363593-1396941 +/hadoop/common/branches/HDFS-347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java:1430995-1467533 +/hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java:1509426-1536569 +/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java:1526848,1541667 +/hadoop/core/branches/branch-0.19/core/src/java/org/apache/hadoop/util/PureJavaCrc32C.java:713112 +/hadoop/core/trunk/src/core/org/apache/hadoop/util/PureJavaCrc32C.java:776175-785643,785929-786278 Modified: commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java URL: http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java?rev=1548947&r1=1548946&r2=1548947&view=diff ============================================================================== --- commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java (original) +++ commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java Sat Dec 7 18:01:01 2013 @@ -21,38 +21,19 @@ package org.apache.commons.compress.comp import static org.junit.Assert.assertArrayEquals; import java.io.BufferedInputStream; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; -import java.util.Arrays; import org.apache.commons.compress.AbstractTestCase; -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorInputStream; import org.apache.commons.compress.utils.IOUtils; public final class FramedSnappyTestCase extends AbstractTestCase { - public void testMatches() throws IOException { - assertFalse(FramedSnappyCompressorInputStream.matches(new byte[10], 10)); - byte[] b = new byte[12]; - final File input = getFile("bla.tar.sz"); - FileInputStream in = new FileInputStream(input); - try { - IOUtils.readFully(in, b); - } finally { - in.close(); - } - assertFalse(FramedSnappyCompressorInputStream.matches(b, 9)); - assertTrue(FramedSnappyCompressorInputStream.matches(b, 10)); - assertTrue(FramedSnappyCompressorInputStream.matches(b, 12)); - } - public void testDefaultExtraction() throws Exception { testUnarchive(new StreamWrapper() { public CompressorInputStream wrap(InputStream is) throws IOException { @@ -79,114 +60,6 @@ public final class FramedSnappyTestCase }); } - /** - * Something big enough to make buffers slide. - */ - public void testLoremIpsum() throws Exception { - final FileInputStream isSz = new FileInputStream(getFile("lorem-ipsum.txt.sz")); - final File outputSz = new File(dir, "lorem-ipsum.1"); - final File outputGz = new File(dir, "lorem-ipsum.2"); - try { - CompressorInputStream in = new FramedSnappyCompressorInputStream(isSz); - FileOutputStream out = null; - try { - out = new FileOutputStream(outputSz); - IOUtils.copy(in, out); - } finally { - if (out != null) { - out.close(); - } - in.close(); - } - final FileInputStream isGz = new FileInputStream(getFile("lorem-ipsum.txt.gz")); - try { - in = new GzipCompressorInputStream(isGz); - try { - out = new FileOutputStream(outputGz); - IOUtils.copy(in, out); - } finally { - if (out != null) { - out.close(); - } - in.close(); - } - } finally { - isGz.close(); - } - } finally { - isSz.close(); - } - - final FileInputStream sz = new FileInputStream(outputSz); - try { - FileInputStream gz = new FileInputStream(outputGz); - try { - assertArrayEquals(IOUtils.toByteArray(sz), - IOUtils.toByteArray(gz)); - } finally { - gz.close(); - } - } finally { - sz.close(); - } - } - - public void testRemainingChunkTypes() throws Exception { - final FileInputStream isSz = new FileInputStream(getFile("mixed.txt.sz")); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - try { - CompressorInputStream in = new FramedSnappyCompressorInputStream(isSz); - IOUtils.copy(in, out); - out.close(); - } finally { - isSz.close(); - } - - assertArrayEquals(new byte[] { '1', '2', '3', '4', - '5', '6', '7', '8', '9', - '5', '6', '7', '8', '9', - '5', '6', '7', '8', '9', - '5', '6', '7', '8', '9', - '5', '6', '7', '8', '9', 10, - '1', '2', '3', '4', - '1', '2', '3', '4', - }, out.toByteArray()); - } - - public void testAvailable() throws Exception { - final FileInputStream isSz = new FileInputStream(getFile("mixed.txt.sz")); - try { - CompressorInputStream in = new FramedSnappyCompressorInputStream(isSz); - assertEquals(0, in.available()); // no chunk read so far - assertEquals('1', in.read()); - assertEquals(3, in.available()); // remainder of first uncompressed block - assertEquals(3, in.read(new byte[5], 0, 3)); - assertEquals('5', in.read()); - assertEquals(4, in.available()); // remainder of literal - assertEquals(4, in.read(new byte[5], 0, 4)); - assertEquals('5', in.read()); - assertEquals(19, in.available()); // remainder of copy - in.close(); - } finally { - isSz.close(); - } - } - - public void testUnskippableChunk() { - byte[] input = new byte[] { - (byte) 0xff, 6, 0, 0, 's', 'N', 'a', 'P', 'p', 'Y', - 2, 2, 0, 0, 1, 1 - }; - try { - CompressorInputStream in = - new FramedSnappyCompressorInputStream(new ByteArrayInputStream(input)); - in.read(); - fail("expected an exception"); - } catch (IOException ex) { - assertTrue(ex.getMessage().indexOf("unskippable chunk") > -1); - } - } - private void testUnarchive(StreamWrapper wrapper) throws Exception { final File input = getFile("bla.tar.sz"); final File output = new File(dir, "bla.tar"); @@ -223,4 +96,5 @@ public final class FramedSnappyTestCase written.close(); } } + } Copied: commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStreamTest.java (from r1548932, commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java) URL: http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStreamTest.java?p2=commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStreamTest.java&p1=commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java&r1=1548932&r2=1548947&rev=1548947&view=diff ============================================================================== --- commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/FramedSnappyTestCase.java (original) +++ commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/snappy/FramedSnappyCompressorInputStreamTest.java Sat Dec 7 18:01:01 2013 @@ -16,11 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.commons.compress.compressors; +package org.apache.commons.compress.compressors.snappy; import static org.junit.Assert.assertArrayEquals; -import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; @@ -32,10 +31,9 @@ import java.util.Arrays; import org.apache.commons.compress.AbstractTestCase; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; -import org.apache.commons.compress.compressors.snappy.FramedSnappyCompressorInputStream; import org.apache.commons.compress.utils.IOUtils; -public final class FramedSnappyTestCase +public final class FramedSnappyCompressorInputStreamTest extends AbstractTestCase { public void testMatches() throws IOException { @@ -53,32 +51,6 @@ public final class FramedSnappyTestCase assertTrue(FramedSnappyCompressorInputStream.matches(b, 12)); } - public void testDefaultExtraction() throws Exception { - testUnarchive(new StreamWrapper() { - public CompressorInputStream wrap(InputStream is) throws IOException { - return new FramedSnappyCompressorInputStream(is); - } - }); - } - - public void testDefaultExtractionViaFactory() throws Exception { - testUnarchive(new StreamWrapper() { - public CompressorInputStream wrap(InputStream is) throws Exception { - return new CompressorStreamFactory() - .createCompressorInputStream(CompressorStreamFactory.SNAPPY_FRAMED, - is); - } - }); - } - - public void testDefaultExtractionViaFactoryAutodetection() throws Exception { - testUnarchive(new StreamWrapper() { - public CompressorInputStream wrap(InputStream is) throws Exception { - return new CompressorStreamFactory().createCompressorInputStream(is); - } - }); - } - /** * Something big enough to make buffers slide. */ @@ -87,7 +59,7 @@ public final class FramedSnappyTestCase final File outputSz = new File(dir, "lorem-ipsum.1"); final File outputGz = new File(dir, "lorem-ipsum.2"); try { - CompressorInputStream in = new FramedSnappyCompressorInputStream(isSz); + InputStream in = new FramedSnappyCompressorInputStream(isSz); FileOutputStream out = null; try { out = new FileOutputStream(outputSz); @@ -135,7 +107,7 @@ public final class FramedSnappyTestCase final FileInputStream isSz = new FileInputStream(getFile("mixed.txt.sz")); ByteArrayOutputStream out = new ByteArrayOutputStream(); try { - CompressorInputStream in = new FramedSnappyCompressorInputStream(isSz); + FramedSnappyCompressorInputStream in = new FramedSnappyCompressorInputStream(isSz); IOUtils.copy(in, out); out.close(); } finally { @@ -156,7 +128,7 @@ public final class FramedSnappyTestCase public void testAvailable() throws Exception { final FileInputStream isSz = new FileInputStream(getFile("mixed.txt.sz")); try { - CompressorInputStream in = new FramedSnappyCompressorInputStream(isSz); + FramedSnappyCompressorInputStream in = new FramedSnappyCompressorInputStream(isSz); assertEquals(0, in.available()); // no chunk read so far assertEquals('1', in.read()); assertEquals(3, in.available()); // remainder of first uncompressed block @@ -178,7 +150,7 @@ public final class FramedSnappyTestCase 2, 2, 0, 0, 1, 1 }; try { - CompressorInputStream in = + FramedSnappyCompressorInputStream in = new FramedSnappyCompressorInputStream(new ByteArrayInputStream(input)); in.read(); fail("expected an exception"); @@ -187,40 +159,21 @@ public final class FramedSnappyTestCase } } - private void testUnarchive(StreamWrapper wrapper) throws Exception { - final File input = getFile("bla.tar.sz"); - final File output = new File(dir, "bla.tar"); - final FileInputStream is = new FileInputStream(input); - try { - // the intermediate BufferedInputStream is there for mark - // support in the autodetection test - final CompressorInputStream in = wrapper.wrap(new BufferedInputStream(is)); - FileOutputStream out = null; - try { - out = new FileOutputStream(output); - IOUtils.copy(in, out); - assertEquals(995, in.getBytesRead()); - } finally { - if (out != null) { - out.close(); - } - in.close(); - } - } finally { - is.close(); - } - final File original = getFile("bla.tar"); - final FileInputStream written = new FileInputStream(output); - try { - FileInputStream orig = new FileInputStream(original); - try { - assertArrayEquals(IOUtils.toByteArray(written), - IOUtils.toByteArray(orig)); - } finally { - orig.close(); - } - } finally { - written.close(); - } + public void testChecksumUnmasking() { + testChecksumUnmasking(0xc757l); + testChecksumUnmasking(0xffffc757l); + } + + public void testChecksumUnmasking(long x) { + assertEquals(Long.toHexString(x), + Long.toHexString(FramedSnappyCompressorInputStream + .unmask(mask(x)))); + } + + private long mask(long x) { + return (((x >>> 15) | (x << 17)) + + FramedSnappyCompressorInputStream.MASK_OFFSET) + & 0xffffFFFFL; } + } Modified: commons/proper/compress/trunk/src/test/resources/mixed.txt.sz URL: http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/test/resources/mixed.txt.sz?rev=1548947&r1=1548946&r2=1548947&view=diff ============================================================================== Binary files - no diff available.