commons-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bode...@apache.org
Subject [2/2] commons-compress git commit: add DEFLATE as compression format to play with
Date Sat, 29 Apr 2017 17:55:05 GMT
add DEFLATE as compression format to play with


Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/b74e5386
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/b74e5386
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/b74e5386

Branch: refs/heads/compress-2.0
Commit: b74e53866767f2323af1ac09681bf80318fe2c49
Parents: 27686ef
Author: Stefan Bodewig <bodewig@apache.org>
Authored: Sat Apr 29 19:54:35 2017 +0200
Committer: Stefan Bodewig <bodewig@apache.org>
Committed: Sat Apr 29 19:54:35 2017 +0200

----------------------------------------------------------------------
 .../formats/deflate/DeflateCompressedInput.java |  92 +++++++++
 .../deflate/DeflateCompressedOutput.java        |  87 +++++++++
 .../deflate/DeflateCompressionFormat.java       |  89 +++++++++
 .../deflate/DeflateCompressionFormatTest.java   |  53 ++++++
 .../formats/deflate/RoundTripTest.java          | 185 +++++++++++++++++++
 .../test-archives/default.tar.deflatez          | Bin 0 -> 468 bytes
 6 files changed, 506 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b74e5386/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedInput.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedInput.java
b/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedInput.java
new file mode 100644
index 0000000..a7d9977
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedInput.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress2.formats.deflate;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.util.zip.Inflater;
+import java.util.zip.InflaterInputStream;
+import org.apache.commons.compress2.compressors.spi.AbstractCompressedInput;
+
+/**
+ * Input for DEFLATE compressed channels.
+ */
+public class DeflateCompressedInput extends AbstractCompressedInput {
+    private final InputStream in;
+    private final Inflater inflater;
+    private boolean nextCalled = false;
+
+    public DeflateCompressedInput(ReadableByteChannel channel) {
+        in = Channels.newInputStream(channel);
+        inflater = new Inflater(false);
+    }
+
+    @Override
+    public ReadableByteChannel next() throws IOException {
+        if (nextCalled) {
+            return null;
+        }
+        nextCalled = true;
+        return Channels.newChannel(new InflaterInputStream(new WrappedStream(in), inflater));
+    }
+
+    @Override
+    public void close() throws IOException {
+        try {
+            in.close();
+        } finally {
+            inflater.end();
+        }
+    }
+
+    private class WrappedStream extends FilterInputStream  {
+        private WrappedStream(InputStream i) {
+            super(i);
+        }
+
+        @Override
+        public void close() {
+            inflater.reset();
+        }
+
+        @Override
+        public int read() throws IOException {
+            int r = super.read();
+            count(r < 0 ? 0 : 1);
+            return r;
+        }
+
+        @Override
+        public int read(byte[] b, int off, int len) throws IOException {
+            int r = super.read(b, off, len);
+            count(r);
+            return r;
+        }
+
+        @Override
+        public long skip(long n) throws IOException {
+            long r = super.skip(n);
+            count(r);
+            return r;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b74e5386/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedOutput.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedOutput.java
b/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedOutput.java
new file mode 100644
index 0000000..f563213
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressedOutput.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress2.formats.deflate;
+
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.WritableByteChannel;
+import java.util.zip.Deflater;
+import java.util.zip.DeflaterOutputStream;
+import org.apache.commons.compress2.compressors.spi.AbstractCompressedOutput;
+
+/**
+ * Output for DEFLATE compressed channels.
+ */
+public class DeflateCompressedOutput extends AbstractCompressedOutput {
+    private WritableByteChannel channel;
+    private final DeflaterOutputStream out;
+    private final Deflater deflater;
+
+    public DeflateCompressedOutput(WritableByteChannel channel) {
+        this.channel = channel;
+        deflater = new Deflater(Deflater.DEFAULT_COMPRESSION, false);
+        out = new DeflaterOutputStream(new WrappedStream(Channels.newOutputStream(channel)),
deflater);
+    }
+
+    @Override
+    public WritableByteChannel startCompressing() {
+        return Channels.newChannel(out);
+    }
+
+    @Override
+    public void finish() throws IOException {
+        out.finish();
+    }
+
+    @Override
+    public void close() throws IOException {
+        try {
+            out.close();
+        } finally {
+            try {
+                deflater.end();
+            } finally {
+                channel.close();
+            }
+        }
+    }
+
+    private class WrappedStream extends FilterOutputStream {
+        private WrappedStream(OutputStream o) {
+            super(o);
+        }
+
+        @Override
+        public void close() { }
+
+        @Override
+        public void write(int b) throws IOException {
+            super.write(b);
+            count(1);
+        }
+
+        @Override
+        public void write(byte[] b, int off, int len) throws IOException {
+            super.write(b, off, len);
+            count(len);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b74e5386/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormat.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormat.java
b/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormat.java
new file mode 100644
index 0000000..ee3bed3
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormat.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress2.formats.deflate;
+
+import java.nio.ByteBuffer;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+import org.apache.commons.compress2.compressors.spi.AbstractCompressionFormat;
+
+/**
+ * Format descriptor for the GZIP format.
+ */
+public class DeflateCompressionFormat extends AbstractCompressionFormat {
+    private static final int MAGIC_1 = 0x78;
+    private static final int MAGIC_2a = 0x01;
+    private static final int MAGIC_2b = 0x5e;
+    private static final int MAGIC_2c = 0x9c;
+    private static final int MAGIC_2d = 0xda;
+
+    /**
+     * "DEFLATE"
+     */
+    public static final String DEFLATE_FORMAT_NAME = "DEFLATE";
+
+    /**
+     * "DEFLATE"
+     */
+    @Override
+    public String getName() {
+        return DEFLATE_FORMAT_NAME;
+    }
+
+    /**
+     * Yes.
+     */
+    @Override
+    public boolean supportsWriting() { return true; }
+
+    /**
+     * Yes.
+     */
+    @Override
+    public boolean supportsAutoDetection() { return true; }
+
+    /**
+     * @return 2
+     */
+    @Override
+    public int getNumberOfBytesRequiredForAutodetection() {
+        return 2;
+    }
+
+    @Override
+    public boolean matches(ByteBuffer probe) {
+        byte[] sig = new byte[2];
+        probe.get(sig);
+        return sig[0] == MAGIC_1 && (
+            sig[1] == (byte) MAGIC_2a ||
+            sig[1] == (byte) MAGIC_2b ||
+            sig[1] == (byte) MAGIC_2c ||
+            sig[1] == (byte) MAGIC_2d);
+    }
+
+    @Override
+    public DeflateCompressedInput readFrom(ReadableByteChannel channel) {
+        return new DeflateCompressedInput(channel);
+    }
+
+    @Override
+    public DeflateCompressedOutput writeTo(WritableByteChannel channel) {
+        return new DeflateCompressedOutput(channel);
+    }
+}

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b74e5386/src/test/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormatTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormatTest.java
b/src/test/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormatTest.java
new file mode 100644
index 0000000..837f31d
--- /dev/null
+++ b/src/test/java/org/apache/commons/compress2/formats/deflate/DeflateCompressionFormatTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress2.formats.deflate;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import org.apache.commons.compress2.util.IOUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class DeflateCompressionFormatTest {
+
+    @Test
+    public void shouldDetectFormat() throws IOException {
+        Assert.assertTrue(isAr("test-archives/default.tar.deflatez"));
+    }
+
+    @Test
+    public void shouldRejectXMLFile() throws IOException {
+        Assert.assertFalse(isAr("test1.xml"));
+    }
+
+
+    private boolean isAr(String file) throws IOException {
+        File f = RoundTripTest.getFile(file);
+        FileInputStream c = new FileInputStream(f);
+        try {
+            byte[] b = new byte[10];
+            IOUtils.readFully(c, b);
+            return new DeflateCompressionFormat().matches(ByteBuffer.wrap(b));
+        } finally {
+            c.close();
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b74e5386/src/test/java/org/apache/commons/compress2/formats/deflate/RoundTripTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress2/formats/deflate/RoundTripTest.java
b/src/test/java/org/apache/commons/compress2/formats/deflate/RoundTripTest.java
new file mode 100644
index 0000000..72f36d4
--- /dev/null
+++ b/src/test/java/org/apache/commons/compress2/formats/deflate/RoundTripTest.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.commons.compress2.formats.deflate;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URL;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+import java.util.Locale;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.commons.compress2.compressors.CompressedInput;
+import org.apache.commons.compress2.compressors.CompressedOutput;
+import org.apache.commons.compress2.util.IOUtils;
+
+public class RoundTripTest {
+
+    private File dir;
+
+    @Before
+    public void createTempDir() throws Exception {
+        dir = mkdir("dir");
+    }
+
+    @After
+    public void removeTempDir() throws Exception {
+        rmdir(dir);
+    }
+
+    @Test
+    public void testRoundtripUsingConstructors() throws Exception {
+        final File output = new File(dir, "constructors.def");
+        final File file1 = getFile("test1.xml");
+        try (WritableByteChannel out = new FileOutputStream(output).getChannel();
+             DeflateCompressedOutput os = new DeflateCompressedOutput(out);
+             WritableByteChannel c = os.startCompressing();
+             ReadableByteChannel in = new FileInputStream(file1).getChannel()) {
+            IOUtils.copy(in, c);
+        }
+
+        final File input = output;
+        final File target = new File(dir, "test1.xml");
+        try (ReadableByteChannel is = new FileInputStream(input).getChannel();
+             DeflateCompressedInput in = new DeflateCompressedInput(is);
+             ReadableByteChannel r = in.next();
+             WritableByteChannel out = new FileOutputStream(target).getChannel()) {
+            IOUtils.copy(r, out);
+        }
+    }
+
+    @Test
+    public void testRoundtripUsingFormatInstanceAndChannels() throws Exception {
+        DeflateCompressionFormat format = new DeflateCompressionFormat();
+        final File output = new File(dir, "format-channels.def");
+        final File file1 = getFile("test1.xml");
+        try (WritableByteChannel out = new FileOutputStream(output).getChannel();
+             DeflateCompressedOutput os = format.writeTo(out);
+             WritableByteChannel c = os.startCompressing();
+             ReadableByteChannel in = new FileInputStream(file1).getChannel()) {
+            IOUtils.copy(in, c);
+        }
+
+        final File input = output;
+        final File target = new File(dir, "test1.xml");
+        try (ReadableByteChannel is = new FileInputStream(input).getChannel();
+             DeflateCompressedInput in = format.readFrom(is);
+             ReadableByteChannel r = in.next();
+             WritableByteChannel out = new FileOutputStream(target).getChannel()) {
+            IOUtils.copy(r, out);
+        }
+    }
+
+    @Test
+    public void testRoundtripUsingFormatInstanceAndPaths() throws Exception {
+        DeflateCompressionFormat format = new DeflateCompressionFormat();
+        final File output = new File(dir, "format-files.def");
+        final File file1 = getFile("test1.xml");
+        try (CompressedOutput os = format.writeTo(output.toPath());
+             WritableByteChannel c = os.startCompressing();
+             ReadableByteChannel in = new FileInputStream(file1).getChannel()) {
+            IOUtils.copy(in, c);
+        }
+
+        final File input = output;
+        final File target = new File(dir, "test1.xml");
+        try (CompressedInput in = format.readFrom(input.toPath());
+             ReadableByteChannel r = in.next();
+             WritableByteChannel out = new FileOutputStream(target).getChannel()) {
+            IOUtils.copy(r, out);
+        }
+    }
+
+    public static File mkdir(String name) throws IOException {
+        File f = File.createTempFile(name, "");
+        f.delete();
+        f.mkdir();
+        return f;
+    }
+
+    public static File getFile(String path) throws IOException {
+        URL url = RoundTripTest.class.getClassLoader().getResource(path);
+        if (url == null) {
+            throw new FileNotFoundException("couldn't find " + path);
+        }
+        URI uri = null;
+        try {
+            uri = url.toURI();
+        } catch (java.net.URISyntaxException ex) {
+            throw new IOException(ex);
+        }
+        return new File(uri);
+    }
+
+    public static void rmdir(File f) {
+        String[] s = f.list();
+        if (s != null) {
+            for (String element : s) {
+                final File file = new File(f, element);
+                if (file.isDirectory()){
+                    rmdir(file);
+                }
+                boolean ok = tryHardToDelete(file);
+                if (!ok && file.exists()){
+                    System.out.println("Failed to delete "+element+" in "+f.getPath());
+                }
+            }
+        }
+        tryHardToDelete(f); // safer to delete and check
+        if (f.exists()){
+            throw new Error("Failed to delete "+f.getPath());
+        }
+    }
+
+    private static final boolean ON_WINDOWS =
+        System.getProperty("os.name").toLowerCase(Locale.ENGLISH)
+        .indexOf("windows") > -1;
+
+    /**
+     * Accommodate Windows bug encountered in both Sun and IBM JDKs.
+     * Others possible. If the delete does not work, call System.gc(),
+     * wait a little and try again.
+     *
+     * @return whether deletion was successful
+     * @since Stolen from FileUtils in Ant 1.8.0
+     */
+    public static boolean tryHardToDelete(File f) {
+        if (f != null && f.exists() && !f.delete()) {
+            if (ON_WINDOWS) {
+                System.gc();
+            }
+            try {
+                Thread.sleep(10);
+            } catch (InterruptedException ex) {
+                // Ignore Exception
+            }
+            return f.delete();
+        }
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b74e5386/src/test/resources/test-archives/default.tar.deflatez
----------------------------------------------------------------------
diff --git a/src/test/resources/test-archives/default.tar.deflatez b/src/test/resources/test-archives/default.tar.deflatez
new file mode 100644
index 0000000..32ebe77
Binary files /dev/null and b/src/test/resources/test-archives/default.tar.deflatez differ


Mime
View raw message