commons-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bode...@apache.org
Subject [commons-compress] 02/05: COMPRESS-477 building a split/spanned zip
Date Sun, 15 Dec 2019 11:10:52 GMT
This is an automated email from the ASF dual-hosted git repository.

bodewig pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-compress.git

commit e61d22ee3549e9ac7379114d9e0a40d68b5dc600
Author: Lee <peteralfredlee@gmail.com>
AuthorDate: Tue Nov 12 09:20:10 2019 +0800

    COMPRESS-477 building a split/spanned zip
    
    add testcases for building a split/spanned zip, as the PR of extracting a split zip is
not merged yet,
    there're no other ways to test in code if my constructed zip is valid.
    
    I will add this part of testcases as soon as the PR of extracting split zip is merged.
---
 .../archivers/zip/ZipArchiveOutputStream.java      |   4 +-
 .../archivers/zip/ZipSplitOutputStream.java        |  29 ++++--
 .../commons/compress/compressors/FileNameUtil.java |   4 +-
 .../commons/compress/archivers/ZipTestCase.java    |  64 +++++++++++++
 .../archivers/zip/ZipSplitOutputStreamTest.java    | 104 +++++++++++++++++++++
 .../zip_to_compare_created_by_zip.zip              | Bin 0 -> 582047 bytes
 6 files changed, 196 insertions(+), 9 deletions(-)

diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
index 962e60b..ebd9c75 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
@@ -1299,6 +1299,8 @@ public class ZipArchiveOutputStream extends ArchiveOutputStream {
                                            final EntryMetaData entryMetaData,
                                            final boolean needsZip64Extra) throws IOException
{
         if(isSplitZip) {
+            // calculate the disk number for every central file header,
+            // this will be used in writing End Of Central Directory and Zip64 End Of Central
Directory
             int currentSplitSegment = ((ZipSplitOutputStream)this.out).getCurrentSplitSegmentIndex();
             if(numberOfCDInDiskData.get(currentSplitSegment) == null) {
                 numberOfCDInDiskData.put(currentSplitSegment, 1);
@@ -1429,7 +1431,7 @@ public class ZipArchiveOutputStream extends ArchiveOutputStream {
      * and {@link Zip64Mode #setUseZip64} is {@link Zip64Mode#Never}.
      */
     protected void writeCentralDirectoryEnd() throws IOException {
-        if(!hasUsedZip64) {
+        if(!hasUsedZip64 && isSplitZip) {
             ((ZipSplitOutputStream)this.out).prepareToWriteUnsplittableContent(eocdLength);
         }
 
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ZipSplitOutputStream.java
b/src/main/java/org/apache/commons/compress/archivers/zip/ZipSplitOutputStream.java
index 80a896d..20ed3c1 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipSplitOutputStream.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipSplitOutputStream.java
@@ -1,3 +1,20 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
 package org.apache.commons.compress.archivers.zip;
 
 import org.apache.commons.compress.compressors.FileNameUtil;
@@ -8,10 +25,11 @@ import java.nio.ByteBuffer;
 public class ZipSplitOutputStream extends OutputStream {
     private OutputStream outputStream;
     private File zipFile;
-    private long splitSize;
+    private final long splitSize;
     private int currentSplitSegmentIndex = 0;
     private long currentSplitSegmentBytesWritten = 0;
     private boolean finished = false;
+    private final byte[] singleByte = new byte[1];
 
     /**
      * 8.5.1 Capacities for split archives are as follows:
@@ -69,8 +87,8 @@ public class ZipSplitOutputStream extends OutputStream {
 
     @Override
     public void write(int i) throws IOException {
-        byte[] b = ByteBuffer.allocate(4).putInt(i).array();
-        write(b);
+        singleByte[0] = (byte)(i & 0xff);
+        write(singleByte);
     }
 
     @Override
@@ -200,11 +218,10 @@ public class ZipSplitOutputStream extends OutputStream {
             extension += newZipSplitSegmentSuffixIndex;
         }
 
-        String newFileName = zipFile.getParent() + File.separatorChar + baseName + extension;
-        File newFile = new File(newFileName);
+        File newFile = new File(zipFile.getParent(), baseName + extension);
 
         if (newFile.exists()) {
-            throw new IOException("split zip segment " + newFileName + " already exists");
+            throw new IOException("split zip segment " + baseName + extension + " already
exists");
         }
         return newFile;
     }
diff --git a/src/main/java/org/apache/commons/compress/compressors/FileNameUtil.java b/src/main/java/org/apache/commons/compress/compressors/FileNameUtil.java
index 570b12e..6f7f557 100644
--- a/src/main/java/org/apache/commons/compress/compressors/FileNameUtil.java
+++ b/src/main/java/org/apache/commons/compress/compressors/FileNameUtil.java
@@ -18,6 +18,7 @@
  */
 package org.apache.commons.compress.compressors;
 
+import java.io.File;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Locale;
@@ -198,8 +199,7 @@ public class FileNameUtil {
             return null;
         }
 
-        int lastSeparatorPosition = Math.max(filename.lastIndexOf('/'), filename.lastIndexOf('\\'));
-        String name = filename.substring(lastSeparatorPosition + 1);
+        String name = new File(filename).getName();
 
         int extensionPosition = name.lastIndexOf('.');
         if(extensionPosition < 0) {
diff --git a/src/test/java/org/apache/commons/compress/archivers/ZipTestCase.java b/src/test/java/org/apache/commons/compress/archivers/ZipTestCase.java
index 6083b41..438c527 100644
--- a/src/test/java/org/apache/commons/compress/archivers/ZipTestCase.java
+++ b/src/test/java/org/apache/commons/compress/archivers/ZipTestCase.java
@@ -647,6 +647,18 @@ public final class ZipTestCase extends AbstractTestCase {
         testInputStreamStatistics("COMPRESS-380/COMPRESS-380.zip", expected);
     }
 
+    @Test
+    public void buildSplitZipTest() throws IOException {
+        File directoryToZip = getFilesToZip();
+        File outputZipFile = new File(dir, "splitZip.zip");
+        long splitSize = 100 * 1024L; /* 100 KB */
+        final ZipArchiveOutputStream zipArchiveOutputStream = new ZipArchiveOutputStream(outputZipFile,
splitSize);
+
+        addFilesToZip(zipArchiveOutputStream, directoryToZip);
+        zipArchiveOutputStream.close();
+        // TODO: validate the created zip files when extracting split zip is merged into
master
+    }
+
     private void testInputStreamStatistics(String fileName, Map<String, List<Long>>
expectedStatistics)
         throws IOException, ArchiveException {
         final File input = getFile(fileName);
@@ -701,4 +713,56 @@ public final class ZipTestCase extends AbstractTestCase {
         final long b = stats.getCompressedCount();
         l.add(Arrays.asList(t, b));
     }
+
+    private File getFilesToZip() throws IOException {
+        File originalZipFile = getFile("COMPRESS-477/split_zip_created_by_zip/zip_to_compare_created_by_zip.zip");
+        ZipFile zipFile = new ZipFile(originalZipFile);
+        Enumeration<ZipArchiveEntry> zipEntries = zipFile.getEntries();
+        ZipArchiveEntry zipEntry;
+        File outputFile;
+        InputStream inputStream;
+        OutputStream outputStream;
+        byte[] buffer;
+        int readLen;
+
+        while (zipEntries.hasMoreElements()) {
+            zipEntry = zipEntries.nextElement();
+            if (zipEntry.isDirectory()) {
+                continue;
+            }
+
+            outputFile = new File(dir, zipEntry.getName());
+            if (!outputFile.getParentFile().exists()) {
+                outputFile.getParentFile().mkdirs();
+            }
+            outputFile = new File(dir, zipEntry.getName());
+
+            inputStream = zipFile.getInputStream(zipEntry);
+            outputStream = new FileOutputStream(outputFile);
+            buffer = new byte[(int)zipEntry.getSize()];
+            while((readLen = inputStream.read(buffer)) > 0) {
+                outputStream.write(buffer, 0, readLen);
+            }
+
+            inputStream.close();
+            outputStream.close();
+        }
+
+        return dir.listFiles()[0];
+    }
+
+    private void addFilesToZip(ZipArchiveOutputStream zipArchiveOutputStream, File fileToAdd)
throws IOException {
+        if(fileToAdd.isDirectory()) {
+            for(File file : fileToAdd.listFiles()) {
+                addFilesToZip(zipArchiveOutputStream, file);
+            }
+        } else {
+            ZipArchiveEntry zipArchiveEntry = new ZipArchiveEntry(fileToAdd.getPath());
+            zipArchiveEntry.setMethod(ZipEntry.DEFLATED);
+
+            zipArchiveOutputStream.putArchiveEntry(zipArchiveEntry);
+            IOUtils.copy(new FileInputStream(fileToAdd), zipArchiveOutputStream);
+            zipArchiveOutputStream.closeArchiveEntry();
+        }
+    }
 }
diff --git a/src/test/java/org/apache/commons/compress/archivers/zip/ZipSplitOutputStreamTest.java
b/src/test/java/org/apache/commons/compress/archivers/zip/ZipSplitOutputStreamTest.java
new file mode 100644
index 0000000..4889e58
--- /dev/null
+++ b/src/test/java/org/apache/commons/compress/archivers/zip/ZipSplitOutputStreamTest.java
@@ -0,0 +1,104 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.commons.compress.archivers.zip;
+
+import org.apache.commons.compress.AbstractTestCase;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+
+public class ZipSplitOutputStreamTest extends AbstractTestCase {
+    @Rule
+    public ExpectedException thrown = ExpectedException.none();
+
+    @Test
+    public void throwsExceptionIfSplitSizeIsTooSmall() throws IOException {
+        thrown.expect(IllegalArgumentException.class);
+        new ZipSplitOutputStream(File.createTempFile("temp", "zip"), (64 * 1024 - 1));
+    }
+
+    @Test
+    public void throwsExceptionIfSplitSizeIsTooLarge() throws IOException {
+        thrown.expect(IllegalArgumentException.class);
+        new ZipSplitOutputStream(File.createTempFile("temp", "zip"), (4 * 1024 * 1024 * 1024L));
+    }
+
+    @Test
+    public void throwsIfUnsplittableSizeLargerThanSplitSize() throws IOException {
+        thrown.expect(IllegalArgumentException.class);
+        long splitSize = 100 * 1024;
+        ZipSplitOutputStream output = new ZipSplitOutputStream(File.createTempFile("temp",
"zip"), splitSize);
+        output.prepareToWriteUnsplittableContent(splitSize + 1);
+    }
+
+    @Test
+    public void splitZipBeginsWithZipSplitSignature() throws IOException {
+        File tempFile = File.createTempFile("temp", "zip");
+        new ZipSplitOutputStream(tempFile, 100 * 1024L);
+
+        InputStream inputStream = new FileInputStream(tempFile);
+        byte[] buffer = new byte[4];
+        inputStream.read(buffer);
+
+        Assert.assertEquals(ByteBuffer.wrap(ZipArchiveOutputStream.DD_SIG).getInt(), ByteBuffer.wrap(buffer).getInt());
+    }
+
+    @Test
+    public void testCreateSplittedFiles() throws IOException {
+        File testOutputFile = new File(dir, "testCreateSplittedFiles.zip");
+        int splitSize = 100 * 1024; /* 100KB */
+        ZipSplitOutputStream zipSplitOutputStream = new ZipSplitOutputStream(testOutputFile,
splitSize);
+
+        File fileToTest = getFile("COMPRESS-477/split_zip_created_by_zip/zip_to_compare_created_by_zip.zip");
+        InputStream inputStream = new FileInputStream(fileToTest);
+        byte[] buffer = new byte[4096];
+        int readLen;
+
+        while ((readLen = inputStream.read(buffer)) > 0) {
+            zipSplitOutputStream.write(buffer, 0, readLen);
+        }
+
+        inputStream.close();
+        zipSplitOutputStream.close();
+
+        File zipFile = new File(dir.getPath(), "testCreateSplittedFiles.z01");
+        Assert.assertEquals(zipFile.length(), splitSize);
+
+        zipFile = new File(dir.getPath(), "testCreateSplittedFiles.z02");
+        Assert.assertEquals(zipFile.length(), splitSize);
+
+        zipFile = new File(dir.getPath(), "testCreateSplittedFiles.z03");
+        Assert.assertEquals(zipFile.length(), splitSize);
+
+        zipFile = new File(dir.getPath(), "testCreateSplittedFiles.z04");
+        Assert.assertEquals(zipFile.length(), splitSize);
+
+        zipFile = new File(dir.getPath(), "testCreateSplittedFiles.z05");
+        Assert.assertEquals(zipFile.length(), splitSize);
+
+        zipFile = new File(dir.getPath(), "testCreateSplittedFiles.zip");
+        Assert.assertEquals(zipFile.length(), (fileToTest.length() + 4 - splitSize * 5));
+    }
+}
diff --git a/src/test/resources/COMPRESS-477/split_zip_created_by_zip/zip_to_compare_created_by_zip.zip
b/src/test/resources/COMPRESS-477/split_zip_created_by_zip/zip_to_compare_created_by_zip.zip
new file mode 100644
index 0000000..b7326ec
Binary files /dev/null and b/src/test/resources/COMPRESS-477/split_zip_created_by_zip/zip_to_compare_created_by_zip.zip
differ


Mime
View raw message