commons-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Gary Gregory <garydgreg...@gmail.com>
Subject Re: [1/2] commons-compress git commit: COMPRESS-423 - Add ZStandard decompression support using Zstd-JNI
Date Tue, 17 Oct 2017 18:58:08 GMT
Should it be "Z_STANDARD" instead of "ZSTANDARD" since "standard" is a word?

Gary

On Tue, Oct 17, 2017 at 12:23 PM, <bodewig@apache.org> wrote:

> Repository: commons-compress
> Updated Branches:
>   refs/heads/master 89bc17055 -> 1c382914c
>
>
> COMPRESS-423 - Add ZStandard decompression support using Zstd-JNI
>
>
> Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
> Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/
> commit/7984387a
> Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/
> tree/7984387a
> Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/
> diff/7984387a
>
> Branch: refs/heads/master
> Commit: 7984387af004fcfe1d1ee12e2c8e6b68f23be001
> Parents: 89bc170
> Author: Andre F de Miranda <trixpan@users.noreply.github.com>
> Authored: Sat Oct 14 17:57:19 2017 +1100
> Committer: Stefan Bodewig <bodewig@apache.org>
> Committed: Tue Oct 17 20:17:01 2017 +0200
>
> ----------------------------------------------------------------------
>  pom.xml                                         |   6 +
>  .../compressors/CompressorStreamFactory.java    |  25 ++-
>  .../zstandard/ZstdCompressorInputStream.java    |  95 +++++++++++
>  .../compressors/zstandard/ZstdUtils.java        |  88 +++++++++++
>  .../ZstdCompressorInputStreamTest.java          | 157 +++++++++++++++++++
>  src/test/resources/bla.tar.zst                  | Bin 0 -> 473 bytes
>  src/test/resources/zstandard.testdata           |   3 +
>  src/test/resources/zstandard.testdata.zst       | Bin 0 -> 94 bytes
>  8 files changed, 372 insertions(+), 2 deletions(-)
> ----------------------------------------------------------------------
>
>
> http://git-wip-us.apache.org/repos/asf/commons-compress/
> blob/7984387a/pom.xml
> ----------------------------------------------------------------------
> diff --git a/pom.xml b/pom.xml
> index 31fc4cd..6a33b38 100644
> --- a/pom.xml
> +++ b/pom.xml
> @@ -74,6 +74,12 @@ jar, tar, zip, dump, 7z, arj.
>        <scope>test</scope>
>      </dependency>
>      <dependency>
> +      <groupId>com.github.luben</groupId>
> +      <artifactId>zstd-jni</artifactId>
> +      <version>1.3.1-1</version>
> +      <optional>true</optional>
> +    </dependency>
> +    <dependency>
>        <groupId>org.brotli</groupId>
>        <artifactId>dec</artifactId>
>        <version>0.1.2</version>
>
> http://git-wip-us.apache.org/repos/asf/commons-compress/
> blob/7984387a/src/main/java/org/apache/commons/compress/compressors/
> CompressorStreamFactory.java
> ----------------------------------------------------------------------
> diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java
> b/src/main/java/org/apache/commons/compress/compressors/
> CompressorStreamFactory.java
> index 4bd22aa..b446963 100644
> --- a/src/main/java/org/apache/commons/compress/compressors/
> CompressorStreamFactory.java
> +++ b/src/main/java/org/apache/commons/compress/compressors/
> CompressorStreamFactory.java
> @@ -55,6 +55,8 @@ import org.apache.commons.compress.compressors.xz.
> XZCompressorInputStream;
>  import org.apache.commons.compress.compressors.xz.
> XZCompressorOutputStream;
>  import org.apache.commons.compress.compressors.xz.XZUtils;
>  import org.apache.commons.compress.compressors.z.ZCompressorInputStream;
> +import org.apache.commons.compress.compressors.zstandard.
> ZstdCompressorInputStream;
> +import org.apache.commons.compress.compressors.zstandard.ZstdUtils;
>  import org.apache.commons.compress.utils.IOUtils;
>  import org.apache.commons.compress.utils.Lists;
>  import org.apache.commons.compress.utils.ServiceLoaderIterator;
> @@ -191,6 +193,14 @@ public class CompressorStreamFactory implements
> CompressorStreamProvider {
>      public static final String LZ4_FRAMED = "lz4-framed";
>
>      /**
> +     * Constant (value {@value}) used to identify the ZStandard
> compression
> +     * algorithm. Not supported as an output stream type.
> +     *
> +     * @since 1.15
> +     */
> +    public static final String ZSTANDARD = "zst";
> +
> +    /**
>       * Constructs a new sorted map from input stream provider names to
> provider
>       * objects.
>       *
> @@ -279,7 +289,7 @@ public class CompressorStreamFactory implements
> CompressorStreamProvider {
>      public static String getBrotli() {
>          return BROTLI;
>      }
> -
> +
>      public static String getBzip2() {
>          return BZIP2;
>      }
> @@ -328,6 +338,10 @@ public class CompressorStreamFactory implements
> CompressorStreamProvider {
>          return LZ4_BLOCK;
>      }
>
> +    public static String getZstandard() {
> +        return ZSTANDARD;
> +    }
> +
>      static void putAll(final Set<String> names, final
> CompressorStreamProvider provider,
>              final TreeMap<String, CompressorStreamProvider> map) {
>          for (final String name : names) {
> @@ -555,6 +569,13 @@ public class CompressorStreamFactory implements
> CompressorStreamProvider {
>                  return new XZCompressorInputStream(in,
> actualDecompressConcatenated, memoryLimitInKb);
>              }
>
> +            if (ZSTANDARD.equalsIgnoreCase(name)) {
> +                if (!ZstdUtils.isZstdCompressionAvailable()) {
> +                    throw new CompressorException("XZ compression is not
> available.");
> +                }
> +                return new ZstdCompressorInputStream(in);
> +            }
> +
>              if (LZMA.equalsIgnoreCase(name)) {
>                  if (!LZMAUtils.isLZMACompressionAvailable()) {
>                      throw new CompressorException("LZMA compression is
> not available");
> @@ -701,7 +722,7 @@ public class CompressorStreamFactory implements
> CompressorStreamProvider {
>      @Override
>      public Set<String> getInputStreamCompressorNames() {
>          return Sets.newHashSet(GZIP, BROTLI, BZIP2, XZ, LZMA, PACK200,
> DEFLATE, SNAPPY_RAW, SNAPPY_FRAMED, Z, LZ4_BLOCK,
> -            LZ4_FRAMED);
> +            LZ4_FRAMED, ZSTANDARD);
>      }
>
>      @Override
>
> http://git-wip-us.apache.org/repos/asf/commons-compress/
> blob/7984387a/src/main/java/org/apache/commons/compress/
> compressors/zstandard/ZstdCompressorInputStream.java
> ----------------------------------------------------------------------
> diff --git a/src/main/java/org/apache/commons/compress/compressors/
> zstandard/ZstdCompressorInputStream.java b/src/main/java/org/apache/
> commons/compress/compressors/zstandard/ZstdCompressorInputStream.java
> new file mode 100644
> index 0000000..1e5dd8d
> --- /dev/null
> +++ b/src/main/java/org/apache/commons/compress/compressors/zstandard/
> ZstdCompressorInputStream.java
> @@ -0,0 +1,95 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements.  See the NOTICE file distributed with
> + * this work for additional information regarding copyright ownership.
> + * The ASF licenses this file to You under the Apache License, Version 2.0
> + * (the "License"); you may not use this file except in compliance with
> + * the License.  You may obtain a copy of the License at
> + *
> + *      http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +package org.apache.commons.compress.compressors.zstandard;
> +
> +
> +import java.io.IOException;
> +import java.io.InputStream;
> +
> +import com.github.luben.zstd.ZstdInputStream;
> +import org.apache.commons.compress.compressors.CompressorInputStream;
> +
> +/**
> + * {@link CompressorInputStream} implementation to decode Zstandard
> encoded stream.
> + * Library relies on <a href="https://github.com/luben/zstd-jni/">Zstandard
> JNI</a>
> + *
> + * @since 1.15
> + */
> +public class ZstdCompressorInputStream extends CompressorInputStream {
> +
> +    private final com.github.luben.zstd.ZstdInputStream decIS;
> +
> +    public ZstdCompressorInputStream(final InputStream in) throws
> IOException {
> +        this.decIS = new ZstdInputStream(in);
> +    }
> +
> +    @Override
> +    public int available() throws IOException {
> +        return decIS.available();
> +    }
> +
> +    @Override
> +    public void close() throws IOException {
> +        decIS.close();
> +    }
> +
> +    @Override
> +    public int read(final byte[] b) throws IOException {
> +        return decIS.read(b);
> +    }
> +
> +    @Override
> +    public long skip(final long n) throws IOException {
> +        return decIS.skip(n);
> +    }
> +
> +    @Override
> +    public void mark(final int readlimit) {
> +        decIS.mark(readlimit);
> +    }
> +
> +    @Override
> +    public boolean markSupported() {
> +        return decIS.markSupported();
> +    }
> +
> +    @Override
> +    public int read() throws IOException {
> +        final int ret = decIS.read();
> +        count(ret == -1 ? 0 : 1);
> +        return ret;
> +    }
> +
> +    @Override
> +    public int read(final byte[] buf, final int off, final int len)
> throws IOException {
> +        final int ret = decIS.read(buf, off, len);
> +        count(ret);
> +        return ret;
> +    }
> +
> +    @Override
> +    public String toString() {
> +        return decIS.toString();
> +    }
> +
> +    @Override
> +    public void reset() throws IOException {
> +        decIS.reset();
> +    }
> +
> +}
> \ No newline at end of file
>
> http://git-wip-us.apache.org/repos/asf/commons-compress/
> blob/7984387a/src/main/java/org/apache/commons/compress/
> compressors/zstandard/ZstdUtils.java
> ----------------------------------------------------------------------
> diff --git a/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdUtils.java
> b/src/main/java/org/apache/commons/compress/compressors/
> zstandard/ZstdUtils.java
> new file mode 100644
> index 0000000..0eb8fa1
> --- /dev/null
> +++ b/src/main/java/org/apache/commons/compress/compressors/
> zstandard/ZstdUtils.java
> @@ -0,0 +1,88 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements.  See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership.  The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License.  You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing,
> + * software distributed under the License is distributed on an
> + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> + * KIND, either express or implied.  See the License for the
> + * specific language governing permissions and limitations
> + * under the License.
> + */
> +package org.apache.commons.compress.compressors.zstandard;
> +
> +/**
> + * Utility code for the Zstandard compression format.
> + * @ThreadSafe
> + * @since 1.14
> + */
> +public class ZstdUtils {
> +
> +    static enum CachedAvailability {
> +        DONT_CACHE, CACHED_AVAILABLE, CACHED_UNAVAILABLE
> +    }
> +
> +    private static volatile CachedAvailability cachedZstdAvailability;
> +
> +    static {
> +        cachedZstdAvailability = CachedAvailability.DONT_CACHE;
> +        try {
> +            Class.forName("org.osgi.framework.BundleEvent");
> +        } catch (final Exception ex) { // NOSONAR
> +            setCacheZstdAvailablity(true);
> +        }
> +    }
> +
> +    /** Private constructor to prevent instantiation of this utility
> class. */
> +    private ZstdUtils() {
> +    }
> +
> +    /**
> +     * Are the classes required to support Zstandard compression
> available?
> +     * @return true if the classes required to support Zstandard
> compression are available
> +     */
> +    public static boolean isZstdCompressionAvailable() {
> +        final CachedAvailability cachedResult = cachedZstdAvailability;
> +        if (cachedResult != CachedAvailability.DONT_CACHE) {
> +            return cachedResult == CachedAvailability.CACHED_AVAILABLE;
> +        }
> +        return internalIsZstdCompressionAvailable();
> +    }
> +
> +    private static boolean internalIsZstdCompressionAvailable() {
> +        try {
> +            Class.forName("com.github.luben.zstd.ZstdInputStream");
> +            return true;
> +        } catch (NoClassDefFoundError | Exception error) {
> +            return false;
> +        }
> +    }
> +
> +    /**
> +     * Whether to cache the result of the Zstandard for Java check.
> +     *
> +     * <p>This defaults to {@code false} in an OSGi environment and
> {@code true} otherwise.</p>
> +     * @param doCache whether to cache the result
> +     */
> +    public static void setCacheZstdAvailablity(final boolean doCache) {
> +        if (!doCache) {
> +            cachedZstdAvailability = CachedAvailability.DONT_CACHE;
> +        } else if (cachedZstdAvailability ==
> CachedAvailability.DONT_CACHE) {
> +            final boolean hasZstd = internalIsZstdCompressionAvailable();
> +            cachedZstdAvailability = hasZstd ? CachedAvailability.CACHED_
> AVAILABLE
> +                : CachedAvailability.CACHED_UNAVAILABLE;
> +        }
> +    }
> +
> +    // only exists to support unit tests
> +    static CachedAvailability getCachedZstdAvailability() {
> +        return cachedZstdAvailability;
> +    }
> +}
>
> http://git-wip-us.apache.org/repos/asf/commons-compress/
> blob/7984387a/src/test/java/org/apache/commons/compress/
> compressors/zstandard/ZstdCompressorInputStreamTest.java
> ----------------------------------------------------------------------
> diff --git a/src/test/java/org/apache/commons/compress/compressors/
> zstandard/ZstdCompressorInputStreamTest.java b/src/test/java/org/apache/
> commons/compress/compressors/zstandard/ZstdCompressorInputStreamTest.java
> new file mode 100644
> index 0000000..5ed276c
> --- /dev/null
> +++ b/src/test/java/org/apache/commons/compress/compressors/zstandard/
> ZstdCompressorInputStreamTest.java
> @@ -0,0 +1,157 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements.  See the NOTICE file distributed with
> + * this work for additional information regarding copyright ownership.
> + * The ASF licenses this file to You under the Apache License, Version 2.0
> + * (the "License"); you may not use this file except in compliance with
> + * the License.  You may obtain a copy of the License at
> + *
> + *      http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +package org.apache.commons.compress.compressors.zstandard;
> +
> +import static org.junit.Assert.assertEquals;
> +import static org.junit.Assert.assertTrue;
> +
> +import java.io.ByteArrayOutputStream;
> +import java.io.File;
> +import java.io.FileInputStream;
> +import java.io.FileOutputStream;
> +import java.io.IOException;
> +import java.io.InputStream;
> +
> +import org.apache.commons.compress.AbstractTestCase;
> +import org.apache.commons.compress.compressors.CompressorInputStream;
> +import org.apache.commons.compress.compressors.CompressorStreamFactory;
> +import org.apache.commons.compress.compressors.zstandard.
> ZstdCompressorInputStream;
> +import org.apache.commons.compress.utils.IOUtils;
> +import org.junit.Assert;
> +import org.junit.Test;
> +
> +public class ZstdCompressorInputStreamTest extends AbstractTestCase {
> +
> +    /**
> +     * Test bridge works fine
> +     * @throws {@link IOException}
> +     */
> +    @Test
> +    public void testZstdDecode() throws IOException {
> +        final File input = getFile("zstandard.testdata.zst");
> +        final File expected = getFile("zstandard.testdata");
> +        try (InputStream inputStream = new FileInputStream(input);
> +            InputStream expectedStream = new FileInputStream(expected);
> +            ZstdCompressorInputStream zstdInputStream = new
> ZstdCompressorInputStream(inputStream)) {
> +            final byte[] b = new byte[97];
> +            IOUtils.readFully(expectedStream, b);
> +            final ByteArrayOutputStream bos = new ByteArrayOutputStream();
> +            int readByte = -1;
> +            while((readByte = zstdInputStream.read()) != -1) {
> +                bos.write(readByte);
> +            }
> +            Assert.assertArrayEquals(b, bos.toByteArray());
> +        }
> +    }
> +
> +    @Test
> +    public void testCachingIsEnabledByDefaultAndZstdUtilsPresent() {
> +        assertEquals(ZstdUtils.CachedAvailability.CACHED_AVAILABLE,
> ZstdUtils.getCachedZstdAvailability());
> +        assertTrue(ZstdUtils.isZstdCompressionAvailable());
> +    }
> +
> +    @Test
> +    public void testCanTurnOffCaching() {
> +        try {
> +            ZstdUtils.setCacheZstdAvailablity(false);
> +            assertEquals(ZstdUtils.CachedAvailability.DONT_CACHE,
> ZstdUtils.getCachedZstdAvailability());
> +            assertTrue(ZstdUtils.isZstdCompressionAvailable());
> +        } finally {
> +            ZstdUtils.setCacheZstdAvailablity(true);
> +        }
> +    }
> +
> +    @Test
> +    public void testTurningOnCachingReEvaluatesAvailability() {
> +        try {
> +            ZstdUtils.setCacheZstdAvailablity(false);
> +            assertEquals(ZstdUtils.CachedAvailability.DONT_CACHE,
> ZstdUtils.getCachedZstdAvailability());
> +            ZstdUtils.setCacheZstdAvailablity(true);
> +            assertEquals(ZstdUtils.CachedAvailability.CACHED_AVAILABLE,
> ZstdUtils.getCachedZstdAvailability());
> +        } finally {
> +            ZstdUtils.setCacheZstdAvailablity(true);
> +        }
> +    }
> +
> +    @Test
> +    public void shouldBeAbleToSkipAByte() throws IOException {
> +        final File input = getFile("zstandard.testdata.zst");
> +        try (InputStream is = new FileInputStream(input)) {
> +            final ZstdCompressorInputStream in =
> +                    new ZstdCompressorInputStream(is);
> +            Assert.assertEquals(1, in.skip(1));
> +            in.close();
> +        }
> +    }
> +
> +    @Test
> +    public void singleByteReadWorksAsExpected() throws IOException {
> +
> +        final File input = getFile("zstandard.testdata.zst");
> +
> +        final File original = getFile("zstandard.testdata");
> +        final long originalFileLength = original.length();
> +
> +        byte[] originalFileContent = new byte[((int) originalFileLength)];
> +
> +        try (InputStream ois = new FileInputStream(original)) {
> +            ois.read(originalFileContent);
> +        }
> +
> +        try (InputStream is = new FileInputStream(input)) {
> +            final ZstdCompressorInputStream in =
> +                    new ZstdCompressorInputStream(is);
> +
> +            Assert.assertEquals(originalFileContent[0], in.read());
> +            in.close();
> +        }
> +    }
> +
> +    @Test
> +    public void singleByteReadReturnsMinusOneAtEof() throws IOException {
> +        final File input = getFile("zstandard.testdata.zst");
> +        try (InputStream is = new FileInputStream(input)) {
> +            final ZstdCompressorInputStream in =
> +                    new ZstdCompressorInputStream(is);
> +            IOUtils.toByteArray(in);
> +            Assert.assertEquals(-1, in.read());
> +            in.close();
> +        }
> +    }
> +
> +    @Test
> +    public void testZstandardUnarchive() throws Exception {
> +        final File input = getFile("bla.tar.zst");
> +        final File output = new File(dir, "bla.tar");
> +        try (InputStream is = new FileInputStream(input)) {
> +            final CompressorInputStream in = new CompressorStreamFactory()
> +                    .createCompressorInputStream("zst", is);
> +            FileOutputStream out = null;
> +            try {
> +                out = new FileOutputStream(output);
> +                IOUtils.copy(in, out);
> +            } finally {
> +                if (out != null) {
> +                    out.close();
> +                }
> +                in.close();
> +            }
> +        }
> +    }
> +
> +}
>
> http://git-wip-us.apache.org/repos/asf/commons-compress/
> blob/7984387a/src/test/resources/bla.tar.zst
> ----------------------------------------------------------------------
> diff --git a/src/test/resources/bla.tar.zst b/src/test/resources/bla.tar.
> zst
> new file mode 100644
> index 0000000..d5fd6e0
> Binary files /dev/null and b/src/test/resources/bla.tar.zst differ
>
> http://git-wip-us.apache.org/repos/asf/commons-compress/
> blob/7984387a/src/test/resources/zstandard.testdata
> ----------------------------------------------------------------------
> diff --git a/src/test/resources/zstandard.testdata b/src/test/resources/
> zstandard.testdata
> new file mode 100644
> index 0000000..e51bfd4
> --- /dev/null
> +++ b/src/test/resources/zstandard.testdata
> @@ -0,0 +1,3 @@
> +And as usual, instead of ipsum lorem we shall just state very clearly:
> +
> +Test test test chocolate
>
> http://git-wip-us.apache.org/repos/asf/commons-compress/
> blob/7984387a/src/test/resources/zstandard.testdata.zst
> ----------------------------------------------------------------------
> diff --git a/src/test/resources/zstandard.testdata.zst
> b/src/test/resources/zstandard.testdata.zst
> new file mode 100644
> index 0000000..86c03fe
> Binary files /dev/null and b/src/test/resources/zstandard.testdata.zst
> differ
>
>

Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message