Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id D1DDC200C11 for ; Sat, 4 Feb 2017 17:19:43 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id D06F5160B63; Sat, 4 Feb 2017 16:19:43 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id AAD8B160B71 for ; Sat, 4 Feb 2017 17:19:41 +0100 (CET) Received: (qmail 29494 invoked by uid 500); 4 Feb 2017 16:19:40 -0000 Mailing-List: contact notifications-help@commons.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@commons.apache.org Delivered-To: mailing list notifications@commons.apache.org Received: (qmail 29438 invoked by uid 99); 4 Feb 2017 16:19:40 -0000 Received: from Unknown (HELO svn01-us-west.apache.org) (209.188.14.144) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 04 Feb 2017 16:19:40 +0000 Received: from svn01-us-west.apache.org (localhost [127.0.0.1]) by svn01-us-west.apache.org (ASF Mail Server at svn01-us-west.apache.org) with ESMTP id 56CAE3A477D for ; Sat, 4 Feb 2017 16:19:39 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1006212 [14/16] - in /websites/production/commons/content/proper/commons-compress: ./ apidocs/org/apache/commons/compress/compressors/lz4/ apidocs/org/apache/commons/compress/compressors/snappy/ apidocs/src-html/org/apache/commons/compress... Date: Sat, 04 Feb 2017 16:19:38 -0000 To: notifications@commons.apache.org From: bodewig@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20170204161939.56CAE3A477D@svn01-us-west.apache.org> archived-at: Sat, 04 Feb 2017 16:19:44 -0000 Modified: websites/production/commons/content/proper/commons-compress/xref/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.html ============================================================================== --- websites/production/commons/content/proper/commons-compress/xref/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.html (original) +++ websites/production/commons/content/proper/commons-compress/xref/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.html Sat Feb 4 16:19:37 2017 @@ -28,263 +28,262 @@ 20 21 import java.io.IOException; 22 import java.io.InputStream; -23 import java.io.PushbackInputStream; -24 import java.util.Arrays; -25 -26 import org.apache.commons.compress.compressors.CompressorInputStream; -27 import org.apache.commons.compress.utils.BoundedInputStream; -28 import org.apache.commons.compress.utils.ByteUtils; -29 import org.apache.commons.compress.utils.IOUtils; -30 -31 /** -32 * CompressorInputStream for the LZ4 frame format. -33 * -34 * Based on the "spec" in the version "1.5.1 (31/03/2015)" -35 * -36 * @see <a href="http://lz4.github.io/lz4/lz4_Frame_format.html">LZ4 Frame Format Description</a> -37 * @since 1.14 -38 * @NotThreadSafe -39 */ -40 public class FramedLZ4CompressorInputStream extends CompressorInputStream { -41 /* -42 * TODO before releasing 1.14: -43 * -44 * + xxhash32 checksum validation -45 * + skippable frames -46 * + decompressConcatenated -47 * + block dependence -48 */ -49 -50 // used by FramedLZ4CompressorOutputStream as well -51 static final byte[] LZ4_SIGNATURE = new byte[] { //NOSONAR -52 4, 0x22, 0x4d, 0x18 -53 }; -54 -55 static final int VERSION_MASK = 0xC0; -56 static final int SUPPORTED_VERSION = 0x40; -57 static final int BLOCK_INDEPENDENCE_MASK = 0x20; -58 static final int BLOCK_CHECKSUM_MASK = 0x10; -59 static final int CONTENT_SIZE_MASK = 0x08; -60 static final int CONTENT_CHECKSUM_MASK = 0x04; -61 static final int BLOCK_MAX_SIZE_MASK = 0x70; -62 static final int UNCOMPRESSED_FLAG_MASK = 0x80000000; -63 -64 // used in no-arg read method -65 private final byte[] oneByte = new byte[1]; -66 -67 private final ByteUtils.ByteSupplier supplier = new ByteUtils.ByteSupplier() { -68 @Override -69 public int getAsByte() throws IOException { -70 return readOneByte(); -71 } -72 }; -73 -74 private final InputStream in; -75 -76 private boolean expectBlockChecksum; -77 private boolean expectContentSize; -78 private boolean expectContentChecksum; -79 -80 private InputStream currentBlock; -81 private boolean endReached, inUncompressed; -82 -83 // used for frame header checksum and content checksum, if present -84 private final XXHash32 contentHash = new XXHash32(); -85 -86 /** -87 * Creates a new input stream that decompresses streams compressed -88 * using the LZ4 frame format. -89 * @param in the InputStream from which to read the compressed data -90 * @throws IOException if reading fails -91 */ -92 public FramedLZ4CompressorInputStream(InputStream in) throws IOException { -93 this.in = in; -94 readSignature(); -95 readFrameDescriptor(); -96 nextBlock(); -97 } -98 -99 /** {@inheritDoc} */ -100 @Override -101 public int read() throws IOException { -102 return read(oneByte, 0, 1) == -1 ? -1 : oneByte[0] & 0xFF; -103 } -104 -105 /** {@inheritDoc} */ -106 @Override -107 public void close() throws IOException { -108 if (currentBlock != null) { -109 currentBlock.close(); -110 currentBlock = null; -111 } -112 in.close(); -113 } -114 -115 /** {@inheritDoc} */ -116 @Override -117 public int read(final byte[] b, final int off, final int len) throws IOException { -118 if (endReached) { -119 return -1; -120 } -121 int r = readOnce(b, off, len); -122 if (r == -1) { -123 nextBlock(); -124 if (!endReached) { -125 r = readOnce(b, off, len); -126 } -127 } -128 if (expectContentChecksum && r != -1) { -129 contentHash.update(b, off, r); -130 } -131 return r; -132 } -133 -134 private void readSignature() throws IOException { -135 final byte[] b = new byte[4]; -136 final int read = IOUtils.readFully(in, b); -137 count(read); -138 if (4 != read || !matches(b, 4)) { -139 throw new IOException("Not a LZ4 frame stream"); -140 } -141 } -142 -143 private void readFrameDescriptor() throws IOException { -144 int flags = readOneByte(); -145 if (flags == -1) { -146 throw new IOException("Premature end of stream while reading frame flags"); -147 } -148 contentHash.update(flags); -149 if ((flags & VERSION_MASK) != SUPPORTED_VERSION) { -150 throw new IOException("Unsupported version " + (flags >> 6)); -151 } -152 if ((flags & BLOCK_INDEPENDENCE_MASK) == 0) { -153 throw new IOException("Block dependence is not supported"); -154 } -155 expectBlockChecksum = (flags & BLOCK_CHECKSUM_MASK) != 0; -156 expectContentSize = (flags & CONTENT_SIZE_MASK) != 0; -157 expectContentChecksum = (flags & CONTENT_CHECKSUM_MASK) != 0; -158 int bdByte = readOneByte(); -159 if (bdByte == -1) { // max size is irrelevant for this implementation -160 throw new IOException("Premature end of stream while reading frame BD byte"); -161 } -162 contentHash.update(bdByte); -163 if (expectContentSize) { // for now we don't care, contains the uncompressed size -164 byte[] contentSize = new byte[8]; -165 int skipped = (int) IOUtils.readFully(in, contentSize); -166 count(skipped); -167 if (8 != skipped) { -168 throw new IOException("Premature end of stream while reading content size"); -169 } -170 contentHash.update(contentSize, 0, contentSize.length); -171 } -172 int headerHash = readOneByte(); -173 if (headerHash == -1) { // partial hash of header. -174 throw new IOException("Premature end of stream while reading frame header checksum"); -175 } -176 int expectedHash = (int) ((contentHash.getValue() >> 8) & 0xff); -177 contentHash.reset(); -178 if (headerHash != expectedHash) { -179 throw new IOException("frame header checksum mismatch."); -180 } -181 } -182 -183 private void nextBlock() throws IOException { -184 maybeFinishCurrentBlock(); -185 long len = ByteUtils.fromLittleEndian(supplier, 4); -186 boolean uncompressed = (len & UNCOMPRESSED_FLAG_MASK) != 0; -187 int realLen = (int) (len & (~UNCOMPRESSED_FLAG_MASK)); -188 if (realLen == 0) { -189 endReached = true; -190 verifyContentChecksum(); -191 return; -192 } -193 InputStream capped = new BoundedInputStream(in, realLen); -194 if (uncompressed) { -195 inUncompressed = true; -196 currentBlock = capped; -197 } else { -198 inUncompressed = false; -199 currentBlock = new BlockLZ4CompressorInputStream(capped); -200 } -201 } -202 -203 private void maybeFinishCurrentBlock() throws IOException { -204 if (currentBlock != null) { -205 currentBlock.close(); -206 currentBlock = null; -207 if (expectBlockChecksum) { -208 int skipped = (int) IOUtils.skip(in, 4); -209 count(skipped); -210 if (4 != skipped) { -211 throw new IOException("Premature end of stream while reading block checksum"); -212 } -213 } -214 } -215 } -216 -217 private void verifyContentChecksum() throws IOException { -218 if (expectContentChecksum) { -219 byte[] checksum = new byte[4]; -220 int read = IOUtils.readFully(in, checksum); -221 count(read); -222 if (4 != read) { -223 throw new IOException("Premature end of stream while reading content checksum"); -224 } -225 long expectedHash = contentHash.getValue(); -226 if (expectedHash != ByteUtils.fromLittleEndian(checksum)) { -227 throw new IOException("content checksum mismatch."); -228 } -229 contentHash.reset(); -230 } -231 } -232 -233 private int readOneByte() throws IOException { -234 final int b = in.read(); -235 if (b != -1) { -236 count(1); -237 return b & 0xFF; -238 } -239 return -1; -240 } -241 -242 private int readOnce(byte[] b, int off, int len) throws IOException { -243 if (inUncompressed) { -244 int cnt = currentBlock.read(b, off, len); -245 count(cnt); -246 return cnt; -247 } else { -248 BlockLZ4CompressorInputStream l = (BlockLZ4CompressorInputStream) currentBlock; -249 long before = l.getBytesRead(); -250 int cnt = currentBlock.read(b, off, len); -251 count(l.getBytesRead() - before); -252 return cnt; -253 } -254 } -255 -256 /** -257 * Checks if the signature matches what is expected for a .lz4 file. -258 * -259 * .lz4 files start with a four byte signature. -260 * -261 * @param signature the bytes to check -262 * @param length the number of bytes to check -263 * @return true if this is a .sz stream, false otherwise -264 */ -265 public static boolean matches(final byte[] signature, final int length) { -266 -267 if (length < LZ4_SIGNATURE.length) { -268 return false; -269 } -270 -271 byte[] shortenedSig = signature; -272 if (signature.length > LZ4_SIGNATURE.length) { -273 shortenedSig = new byte[LZ4_SIGNATURE.length]; -274 System.arraycopy(signature, 0, shortenedSig, 0, LZ4_SIGNATURE.length); -275 } -276 -277 return Arrays.equals(shortenedSig, LZ4_SIGNATURE); -278 } -279 } +23 import java.util.Arrays; +24 +25 import org.apache.commons.compress.compressors.CompressorInputStream; +26 import org.apache.commons.compress.utils.BoundedInputStream; +27 import org.apache.commons.compress.utils.ByteUtils; +28 import org.apache.commons.compress.utils.IOUtils; +29 +30 /** +31 * CompressorInputStream for the LZ4 frame format. +32 * +33 * Based on the "spec" in the version "1.5.1 (31/03/2015)" +34 * +35 * @see <a href="http://lz4.github.io/lz4/lz4_Frame_format.html">LZ4 Frame Format Description</a> +36 * @since 1.14 +37 * @NotThreadSafe +38 */ +39 public class FramedLZ4CompressorInputStream extends CompressorInputStream { +40 /* +41 * TODO before releasing 1.14: +42 * +43 * + xxhash32 checksum validation +44 * + skippable frames +45 * + decompressConcatenated +46 * + block dependence +47 */ +48 +49 // used by FramedLZ4CompressorOutputStream as well +50 static final byte[] LZ4_SIGNATURE = new byte[] { //NOSONAR +51 4, 0x22, 0x4d, 0x18 +52 }; +53 +54 static final int VERSION_MASK = 0xC0; +55 static final int SUPPORTED_VERSION = 0x40; +56 static final int BLOCK_INDEPENDENCE_MASK = 0x20; +57 static final int BLOCK_CHECKSUM_MASK = 0x10; +58 static final int CONTENT_SIZE_MASK = 0x08; +59 static final int CONTENT_CHECKSUM_MASK = 0x04; +60 static final int BLOCK_MAX_SIZE_MASK = 0x70; +61 static final int UNCOMPRESSED_FLAG_MASK = 0x80000000; +62 +63 // used in no-arg read method +64 private final byte[] oneByte = new byte[1]; +65 +66 private final ByteUtils.ByteSupplier supplier = new ByteUtils.ByteSupplier() { +67 @Override +68 public int getAsByte() throws IOException { +69 return readOneByte(); +70 } +71 }; +72 +73 private final InputStream in; +74 +75 private boolean expectBlockChecksum; +76 private boolean expectContentSize; +77 private boolean expectContentChecksum; +78 +79 private InputStream currentBlock; +80 private boolean endReached, inUncompressed; +81 +82 // used for frame header checksum and content checksum, if present +83 private final XXHash32 contentHash = new XXHash32(); +84 +85 /** +86 * Creates a new input stream that decompresses streams compressed +87 * using the LZ4 frame format. +88 * @param in the InputStream from which to read the compressed data +89 * @throws IOException if reading fails +90 */ +91 public FramedLZ4CompressorInputStream(InputStream in) throws IOException { +92 this.in = in; +93 readSignature(); +94 readFrameDescriptor(); +95 nextBlock(); +96 } +97 +98 /** {@inheritDoc} */ +99 @Override +100 public int read() throws IOException { +101 return read(oneByte, 0, 1) == -1 ? -1 : oneByte[0] & 0xFF; +102 } +103 +104 /** {@inheritDoc} */ +105 @Override +106 public void close() throws IOException { +107 if (currentBlock != null) { +108 currentBlock.close(); +109 currentBlock = null; +110 } +111 in.close(); +112 } +113 +114 /** {@inheritDoc} */ +115 @Override +116 public int read(final byte[] b, final int off, final int len) throws IOException { +117 if (endReached) { +118 return -1; +119 } +120 int r = readOnce(b, off, len); +121 if (r == -1) { +122 nextBlock(); +123 if (!endReached) { +124 r = readOnce(b, off, len); +125 } +126 } +127 if (expectContentChecksum && r != -1) { +128 contentHash.update(b, off, r); +129 } +130 return r; +131 } +132 +133 private void readSignature() throws IOException { +134 final byte[] b = new byte[4]; +135 final int read = IOUtils.readFully(in, b); +136 count(read); +137 if (4 != read || !matches(b, 4)) { +138 throw new IOException("Not a LZ4 frame stream"); +139 } +140 } +141 +142 private void readFrameDescriptor() throws IOException { +143 int flags = readOneByte(); +144 if (flags == -1) { +145 throw new IOException("Premature end of stream while reading frame flags"); +146 } +147 contentHash.update(flags); +148 if ((flags & VERSION_MASK) != SUPPORTED_VERSION) { +149 throw new IOException("Unsupported version " + (flags >> 6)); +150 } +151 if ((flags & BLOCK_INDEPENDENCE_MASK) == 0) { +152 throw new IOException("Block dependence is not supported"); +153 } +154 expectBlockChecksum = (flags & BLOCK_CHECKSUM_MASK) != 0; +155 expectContentSize = (flags & CONTENT_SIZE_MASK) != 0; +156 expectContentChecksum = (flags & CONTENT_CHECKSUM_MASK) != 0; +157 int bdByte = readOneByte(); +158 if (bdByte == -1) { // max size is irrelevant for this implementation +159 throw new IOException("Premature end of stream while reading frame BD byte"); +160 } +161 contentHash.update(bdByte); +162 if (expectContentSize) { // for now we don't care, contains the uncompressed size +163 byte[] contentSize = new byte[8]; +164 int skipped = (int) IOUtils.readFully(in, contentSize); +165 count(skipped); +166 if (8 != skipped) { +167 throw new IOException("Premature end of stream while reading content size"); +168 } +169 contentHash.update(contentSize, 0, contentSize.length); +170 } +171 int headerHash = readOneByte(); +172 if (headerHash == -1) { // partial hash of header. +173 throw new IOException("Premature end of stream while reading frame header checksum"); +174 } +175 int expectedHash = (int) ((contentHash.getValue() >> 8) & 0xff); +176 contentHash.reset(); +177 if (headerHash != expectedHash) { +178 throw new IOException("frame header checksum mismatch."); +179 } +180 } +181 +182 private void nextBlock() throws IOException { +183 maybeFinishCurrentBlock(); +184 long len = ByteUtils.fromLittleEndian(supplier, 4); +185 boolean uncompressed = (len & UNCOMPRESSED_FLAG_MASK) != 0; +186 int realLen = (int) (len & (~UNCOMPRESSED_FLAG_MASK)); +187 if (realLen == 0) { +188 endReached = true; +189 verifyContentChecksum(); +190 return; +191 } +192 InputStream capped = new BoundedInputStream(in, realLen); +193 if (uncompressed) { +194 inUncompressed = true; +195 currentBlock = capped; +196 } else { +197 inUncompressed = false; +198 currentBlock = new BlockLZ4CompressorInputStream(capped); +199 } +200 } +201 +202 private void maybeFinishCurrentBlock() throws IOException { +203 if (currentBlock != null) { +204 currentBlock.close(); +205 currentBlock = null; +206 if (expectBlockChecksum) { +207 int skipped = (int) IOUtils.skip(in, 4); +208 count(skipped); +209 if (4 != skipped) { +210 throw new IOException("Premature end of stream while reading block checksum"); +211 } +212 } +213 } +214 } +215 +216 private void verifyContentChecksum() throws IOException { +217 if (expectContentChecksum) { +218 byte[] checksum = new byte[4]; +219 int read = IOUtils.readFully(in, checksum); +220 count(read); +221 if (4 != read) { +222 throw new IOException("Premature end of stream while reading content checksum"); +223 } +224 long expectedHash = contentHash.getValue(); +225 if (expectedHash != ByteUtils.fromLittleEndian(checksum)) { +226 throw new IOException("content checksum mismatch."); +227 } +228 contentHash.reset(); +229 } +230 } +231 +232 private int readOneByte() throws IOException { +233 final int b = in.read(); +234 if (b != -1) { +235 count(1); +236 return b & 0xFF; +237 } +238 return -1; +239 } +240 +241 private int readOnce(byte[] b, int off, int len) throws IOException { +242 if (inUncompressed) { +243 int cnt = currentBlock.read(b, off, len); +244 count(cnt); +245 return cnt; +246 } else { +247 BlockLZ4CompressorInputStream l = (BlockLZ4CompressorInputStream) currentBlock; +248 long before = l.getBytesRead(); +249 int cnt = currentBlock.read(b, off, len); +250 count(l.getBytesRead() - before); +251 return cnt; +252 } +253 } +254 +255 /** +256 * Checks if the signature matches what is expected for a .lz4 file. +257 * +258 * .lz4 files start with a four byte signature. +259 * +260 * @param signature the bytes to check +261 * @param length the number of bytes to check +262 * @return true if this is a .sz stream, false otherwise +263 */ +264 public static boolean matches(final byte[] signature, final int length) { +265 +266 if (length < LZ4_SIGNATURE.length) { +267 return false; +268 } +269 +270 byte[] shortenedSig = signature; +271 if (signature.length > LZ4_SIGNATURE.length) { +272 shortenedSig = new byte[LZ4_SIGNATURE.length]; +273 System.arraycopy(signature, 0, shortenedSig, 0, LZ4_SIGNATURE.length); +274 } +275 +276 return Arrays.equals(shortenedSig, LZ4_SIGNATURE); +277 } +278 }