commons-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bode...@apache.org
Subject [1/2] commons-compress git commit: COMPRESS-375 Allow the deferred creation of `ZipArchiveEntry` for parallel zips
Date Sun, 04 Dec 2016 11:24:13 GMT
Repository: commons-compress
Updated Branches:
  refs/heads/master 9238eab67 -> 66726eabf


COMPRESS-375 Allow the deferred creation of `ZipArchiveEntry` for parallel zips

In some cases when creating parallel zip archive the `ZipArchiveEntry`
to be added could not be created before the `InputStream` is read.
In those cases there is no point in passing `ZipArchiveEntry` and
`InputStreamSupplier` as you can't actually defer the creation
of the `InputStream` as it's needed for the `ZipArchiveEntry`.

Add `ZipArchiveEntryRequestSupplier` to allow the deferred
creation of both `ZipArchiveEntry` and `InputStream`.


Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/fdbfa5d9
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/fdbfa5d9
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/fdbfa5d9

Branch: refs/heads/master
Commit: fdbfa5d962dbc31a2c77fc3d2d5e0f07b36f732f
Parents: 9238eab
Author: Plamen Totev <plamenttv@gmail.com>
Authored: Sat Dec 3 19:17:09 2016 +0200
Committer: Stefan Bodewig <bodewig@apache.org>
Committed: Sun Dec 4 12:21:06 2016 +0100

----------------------------------------------------------------------
 .../zip/ParallelScatterZipCreator.java          | 38 +++++++++++++++++++
 .../zip/ZipArchiveEntryRequestSupplier.java     | 40 ++++++++++++++++++++
 .../zip/ParallelScatterZipCreatorTest.java      | 32 ++++++++++++++--
 3 files changed, 106 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/commons-compress/blob/fdbfa5d9/src/main/java/org/apache/commons/compress/archivers/zip/ParallelScatterZipCreator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ParallelScatterZipCreator.java
b/src/main/java/org/apache/commons/compress/archivers/zip/ParallelScatterZipCreator.java
index de437b1..6d407e3 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ParallelScatterZipCreator.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ParallelScatterZipCreator.java
@@ -137,6 +137,19 @@ public class ParallelScatterZipCreator {
     }
 
     /**
+     * Adds an archive entry to this archive.
+     * <p>
+     * This method is expected to be called from a single client thread
+     * </p>
+     *
+     * @param zipArchiveEntryRequestSupplier Should supply the entry to be added.
+     * @since 1.13
+     */
+    public void addArchiveEntry(final ZipArchiveEntryRequestSupplier zipArchiveEntryRequestSupplier)
{
+        submit(createCallable(zipArchiveEntryRequestSupplier));
+    }
+
+    /**
      * Submit a callable for compression.
      *
      * @see ParallelScatterZipCreator#createCallable for details of if/when to use this.
@@ -180,6 +193,31 @@ public class ParallelScatterZipCreator {
         };
     }
 
+    /**
+     * Create a callable that will compress archive entry supplied by {@link ZipArchiveEntryRequestSupplier}.
+     *
+     * <p>This method is expected to be called from a single client thread.</p>
+     *
+     * The same as {@link #createCallable(ZipArchiveEntry, InputStreamSupplier)}, but the
archive entry
+     * to be added is supplied by a {@link ZipArchiveEntryRequestSupplier}.
+     *
+     * @see #createCallable(ZipArchiveEntry, InputStreamSupplier)
+     *
+     * @param zipArchiveEntryRequestSupplier Should supply the entry to be added.
+     * @return A callable that should subsequently passed to #submit, possibly in a wrapped/adapted
from. The
+     * value of this callable is not used, but any exceptions happening inside the compression
+     * will be propagated through the callable.
+     * @since 1.13
+     */
+    public final Callable<Object> createCallable(final ZipArchiveEntryRequestSupplier
zipArchiveEntryRequestSupplier) {
+        return new Callable<Object>() {
+            @Override
+            public Object call() throws Exception {
+                tlScatterStreams.get().addArchiveEntry(zipArchiveEntryRequestSupplier.get());
+                return null;
+            }
+        };
+    }
 
     /**
      * Write the contents this to the target {@link ZipArchiveOutputStream}.

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/fdbfa5d9/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntryRequestSupplier.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntryRequestSupplier.java
b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntryRequestSupplier.java
new file mode 100644
index 0000000..e213fef
--- /dev/null
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveEntryRequestSupplier.java
@@ -0,0 +1,40 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.commons.compress.archivers.zip;
+
+import org.apache.commons.compress.parallel.InputStreamSupplier;
+
+import java.io.InputStream;
+
+/**
+ * Supplies {@link ZipArchiveEntryRequest}.
+ *
+ * Implementations are required to support thread-handover. While an instance will
+ * not be accessed concurrently by multiple threads, it will be called by
+ * a different thread than it was created on.
+ *
+ * @since 1.13
+ */
+public interface ZipArchiveEntryRequestSupplier {
+
+    /**
+     * Supply a {@link ZipArchiveEntryRequest} to be added to a parallel archive.
+     * @return The {@link ZipArchiveEntryRequest} instance. Should never be null.
+     */
+    ZipArchiveEntryRequest get();
+}

http://git-wip-us.apache.org/repos/asf/commons-compress/blob/fdbfa5d9/src/test/java/org/apache/commons/compress/archivers/zip/ParallelScatterZipCreatorTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/compress/archivers/zip/ParallelScatterZipCreatorTest.java
b/src/test/java/org/apache/commons/compress/archivers/zip/ParallelScatterZipCreatorTest.java
index b09da00..efa38a3 100644
--- a/src/test/java/org/apache/commons/compress/archivers/zip/ParallelScatterZipCreatorTest.java
+++ b/src/test/java/org/apache/commons/compress/archivers/zip/ParallelScatterZipCreatorTest.java
@@ -113,12 +113,23 @@ public class ParallelScatterZipCreatorTest {
         for (int i = 0; i < NUMITEMS; i++){
             final byte[] payloadBytes = ("content" + i).getBytes();
             final ZipArchiveEntry za = createZipArchiveEntry(entries, i, payloadBytes);
-            zipCreator.addArchiveEntry(za, new InputStreamSupplier() {
+            final InputStreamSupplier iss = new InputStreamSupplier() {
                 @Override
                 public InputStream get() {
                     return new ByteArrayInputStream(payloadBytes);
                 }
-            });
+            };
+            if (i % 2 == 0) {
+                zipCreator.addArchiveEntry(za, iss);
+            } else {
+                final ZipArchiveEntryRequestSupplier zaSupplier = new ZipArchiveEntryRequestSupplier()
{
+                    @Override
+                    public ZipArchiveEntryRequest get() {
+                        return ZipArchiveEntryRequest.createZipArchiveEntryRequest(za, iss);
+                    }
+                };
+                zipCreator.addArchiveEntry(zaSupplier);
+            }
         }
         return entries;
     }
@@ -128,12 +139,25 @@ public class ParallelScatterZipCreatorTest {
         for (int i = 0; i < NUMITEMS; i++){
             final byte[] payloadBytes = ("content" + i).getBytes();
             final ZipArchiveEntry za = createZipArchiveEntry(entries, i, payloadBytes);
-            final Callable<Object> callable = zipCreator.createCallable(za, new InputStreamSupplier()
{
+            final InputStreamSupplier iss = new InputStreamSupplier() {
                 @Override
                 public InputStream get() {
                     return new ByteArrayInputStream(payloadBytes);
                 }
-            });
+            };
+            final Callable<Object> callable;
+            if (i % 2 == 0) {
+                callable = zipCreator.createCallable(za, iss);
+            } else {
+                final ZipArchiveEntryRequestSupplier zaSupplier = new ZipArchiveEntryRequestSupplier()
{
+                    @Override
+                    public ZipArchiveEntryRequest get() {
+                        return ZipArchiveEntryRequest.createZipArchiveEntryRequest(za, iss);
+                    }
+                };
+                callable = zipCreator.createCallable(zaSupplier);
+            }
+
             zipCreator.submit(callable);
         }
         return entries;


Mime
View raw message