commons-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bode...@apache.org
Subject svn commit: r1152430 - in /commons/proper/compress/branches/zip64/src: main/java/org/apache/commons/compress/archivers/zip/ test/java/org/apache/commons/compress/archivers/zip/
Date Sat, 30 Jul 2011 06:17:25 GMT
Author: bodewig
Date: Sat Jul 30 06:17:24 2011
New Revision: 1152430

URL: http://svn.apache.org/viewvc?rev=1152430&view=rev
Log:
ZIP64 support for writing big archive entries in the most simple of the seven possible permutations:
no compression + stream + size known upfront.  COMPRESS-150

Modified:
    commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
    commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipEightByteInteger.java
    commons/proper/compress/branches/zip64/src/test/java/org/apache/commons/compress/archivers/zip/Zip64SupportTest.java

Modified: commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
URL: http://svn.apache.org/viewvc/commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java?rev=1152430&r1=1152429&r2=1152430&view=diff
==============================================================================
--- commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
(original)
+++ commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
Sat Jul 30 06:17:24 2011
@@ -180,11 +180,6 @@ public class ZipArchiveOutputStream exte
     private static final byte[] LZERO = {0, 0, 0, 0};
 
     /**
-     * Helper, a 0 as ZipEightByteInteger.
-     */
-    private static final byte[] DLZERO = {0, 0, 0, 0, 0, 0, 0, 0};
-
-    /**
      * Holds the offsets of the LFH starts for each entry.
      */
     private final Map<ZipArchiveEntry, Long> offsets =
@@ -484,6 +479,26 @@ public class ZipArchiveOutputStream exte
             entry.setCompressedSize(entry.getSize());
         }
 
+        // add a ZIP64 extended information extra field if we already
+        // know it is going to be needed
+        if (entry.getSize() >= ZIP64_MAGIC
+            || entry.getCompressedSize() >= ZIP64_MAGIC) {
+
+            Zip64ExtendedInformationExtraField z64 = getZip64Extra(entry);
+            if (entry.getMethod() == STORED) {
+                ZipEightByteInteger size =
+                    new ZipEightByteInteger(entry.getSize());
+                z64.setSize(size);
+                z64.setCompressedSize(size);
+            } else {
+                // just a placeholder, real data will be in data
+                // descriptor or inserted later via RandomAccessFile
+                z64.setSize(ZipEightByteInteger.ZERO);
+                z64.setCompressedSize(ZipEightByteInteger.ZERO);
+            }
+            entry.setExtra();
+        }
+
         if (entry.getMethod() == DEFLATED && hasCompressionLevelChanged) {
             def.setLevel(level);
             hasCompressionLevelChanged = false;
@@ -708,7 +723,7 @@ public class ZipArchiveOutputStream exte
         writeVersionNeededToExtractAndGeneralPurposeBits(zipMethod,
                                                          !encodable
                                                          && fallbackToUTF8,
-                                                         false);
+                                                         hasZip64Extra(ze));
         written += WORD;
 
         // compression method
@@ -729,8 +744,9 @@ public class ZipArchiveOutputStream exte
             writeOut(LZERO);
         } else {
             writeOut(ZipLong.getBytes(ze.getCrc()));
-            writeOut(ZipLong.getBytes(ze.getSize()));
-            writeOut(ZipLong.getBytes(ze.getSize()));
+            byte[] size = ZipLong.getBytes(Math.min(ze.getSize(), ZIP64_MAGIC));
+            writeOut(size);
+            writeOut(size);
         }
         // CheckStyle:MagicNumber OFF
         written += 12;
@@ -1122,8 +1138,14 @@ public class ZipArchiveOutputStream exte
             ze.getExtraField(Zip64ExtendedInformationExtraField
                              .HEADER_ID);
         if (z64 == null) {
+            /*
+            System.err.println("Adding z64 for " + ze.getName()
+                               + ", method: " + ze.getMethod()
+                               + " (" + (ze.getMethod() == STORED) + ")"
+                               + ", raf: " + (raf != null));
+            */
             z64 = new Zip64ExtendedInformationExtraField();
-            ze.addExtraField(z64);
+            ze.addAsFirstExtraField(z64);
         }
         return z64;
     }

Modified: commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipEightByteInteger.java
URL: http://svn.apache.org/viewvc/commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipEightByteInteger.java?rev=1152430&r1=1152429&r2=1152430&view=diff
==============================================================================
--- commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipEightByteInteger.java
(original)
+++ commons/proper/compress/branches/zip64/src/main/java/org/apache/commons/compress/archivers/zip/ZipEightByteInteger.java
Sat Jul 30 06:17:24 2011
@@ -63,6 +63,8 @@ public final class ZipEightByteInteger {
 
     private final BigInteger value;
 
+    public static ZipEightByteInteger ZERO = new ZipEightByteInteger(0);
+
     /**
      * Create instance from a number.
      * @param value the long to store as a ZipEightByteInteger

Modified: commons/proper/compress/branches/zip64/src/test/java/org/apache/commons/compress/archivers/zip/Zip64SupportTest.java
URL: http://svn.apache.org/viewvc/commons/proper/compress/branches/zip64/src/test/java/org/apache/commons/compress/archivers/zip/Zip64SupportTest.java?rev=1152430&r1=1152429&r2=1152430&view=diff
==============================================================================
--- commons/proper/compress/branches/zip64/src/test/java/org/apache/commons/compress/archivers/zip/Zip64SupportTest.java
(original)
+++ commons/proper/compress/branches/zip64/src/test/java/org/apache/commons/compress/archivers/zip/Zip64SupportTest.java
Sat Jul 30 06:17:24 2011
@@ -317,7 +317,6 @@ public class Zip64SupportTest {
                     a.close();
                 }
             }
-
         };
 
     @Ignore
@@ -334,6 +333,165 @@ public class Zip64SupportTest {
                              false);
     }
 
+    /*
+     * One entry of length 5 billion bytes, written without
+     * compression to a stream.
+     *
+     * No Compression + Stream => sizes must be known before data is
+     * written and are stored directly inside the LFH.  No Data
+     * Descriptor at all.
+     *
+     * Creates a temporary archive of approx 5GB in size
+     */
+    @Test public void writeBigStoredEntryToStream() throws Throwable {
+        withTemporaryArchive("writeBigStoredEntryToStream",
+                             new ZipOutputTest() {
+                                 public void test(File f,
+                                                  ZipArchiveOutputStream zos)
+                                     throws IOException {
+                                     byte[] buf = new byte[1000 * 1000];
+                                     ZipArchiveEntry zae =
+                                         new ZipArchiveEntry("0");
+                                     zae.setSize(FIVE_BILLION);
+                                     zae.setMethod(ZipArchiveEntry.STORED);
+                                     zae.setCrc(0x5c316f50L);
+                                     zos.putArchiveEntry(zae);
+                                     for (int j = 0;
+                                          j < FIVE_BILLION / 1000 / 1000;
+                                          j++) {
+                                         zos.write(buf);
+                                     }
+                                     zos.closeArchiveEntry();
+                                     zos.close();
+
+                                     RandomAccessFile a =
+                                         new RandomAccessFile(f, "r");
+                                     try {
+                                         final long end = a.length();
+                                         long cdOffsetLoc = end - 22
+                                             - 20
+                                             - 56
+                                             + 48;
+                                         // seek to central directory
+                                         a.seek(cdOffsetLoc);
+                                         byte[] cdOffset = new byte[8];
+                                         a.readFully(cdOffset);
+                                         a.seek(ZipEightByteInteger
+                                                .getLongValue(cdOffset));
+
+                                         // grab first entry, verify
+                                         // sizes are 0xFFFFFFFF and
+                                         // it has a ZIP64 extended
+                                         // information extra field
+                                         byte[] header = new byte[8];
+                                         a.readFully(header);
+                                         assertArrayEquals(new byte[] {
+                                                 // sig
+                                                 (byte) 0x50, (byte) 0x4b, 1, 2,
+                                                 // version made by
+                                                 45, 0,
+                                                 // version needed to extract
+                                                 45, 0,
+                                             }, header);
+                                         // ignore GPB, method, timestamp
+                                         a.skipBytes(8);
+                                         byte[] rest = new byte[31];
+                                         a.readFully(rest);
+                                         assertArrayEquals(new byte[] {
+                                                 // CRC
+                                                 (byte) 0x50, (byte) 0x6F,
+                                                 (byte) 0x31, (byte) 0x5c,
+                                                 // Compressed Size
+                                                 (byte) 0xFF, (byte) 0xFF,
+                                                 (byte) 0xFF, (byte) 0xFF,
+                                                 // Original Size
+                                                 (byte) 0xFF, (byte) 0xFF,
+                                                 (byte) 0xFF, (byte) 0xFF,
+                                                 // file name length
+                                                 1, 0,
+                                                 // extra field length
+                                                 20, 0,
+                                                 // comment length
+                                                 0, 0,
+                                                 // disk number
+                                                 0, 0,
+                                                 // attributes
+                                                 0, 0,
+                                                 0, 0, 0, 0,
+                                                 // offset
+                                                 0, 0, 0, 0,
+                                                 // file name
+                                                 (byte) '0'
+                                             }, rest);
+                                         byte[] extra = new byte[20];
+                                         a.readFully(extra);
+                                         // 5e9 == 0x12A05F200
+                                         assertArrayEquals(new byte[] {
+                                                 // Header-ID
+                                                 1, 0,
+                                                 // size of extra
+                                                 16, 0,
+                                                 // original size
+                                                 0, (byte) 0xF2, 5, (byte) 0x2A,
+                                                 1, 0, 0, 0,
+                                                 // compressed size
+                                                 0, (byte) 0xF2, 5, (byte) 0x2A,
+                                                 1, 0, 0, 0,
+                                             }, extra);
+
+                                         // and now validate local file header
+                                         a.seek(0);
+                                         header = new byte[6];
+                                         a.readFully(header);
+                                         assertArrayEquals(new byte[] {
+                                                 // sig
+                                                 (byte) 0x50, (byte) 0x4b, 3, 4,
+                                                 // version needed to extract
+                                                 45, 0,
+                                             }, header);
+                                         // ignore GPB, method, timestamp
+                                         a.skipBytes(8);
+                                         rest = new byte[17];
+                                         a.readFully(rest);
+                                         assertArrayEquals(new byte[] {
+                                                 // CRC
+                                                 (byte) 0x50, (byte) 0x6F,
+                                                 (byte) 0x31, (byte) 0x5c,
+                                                 // Compressed Size
+                                                 (byte) 0xFF, (byte) 0xFF,
+                                                 (byte) 0xFF, (byte) 0xFF,
+                                                 // Original Size
+                                                 (byte) 0xFF, (byte) 0xFF,
+                                                 (byte) 0xFF, (byte) 0xFF,
+                                                 // file name length
+                                                 1, 0,
+                                                 // extra field length
+                                                 20, 0,
+                                                 // file name
+                                                 (byte) '0'
+                                             }, rest);
+                                         a.readFully(extra);
+                                         // 5e9 == 0x12A05F200
+                                         assertArrayEquals(new byte[] {
+                                                 // Header-ID
+                                                 1, 0,
+                                                 // size of extra
+                                                 16, 0,
+                                                 // original size
+                                                 0, (byte) 0xF2, 5, (byte) 0x2A,
+                                                 1, 0, 0, 0,
+                                                 // compressed size
+                                                 0, (byte) 0xF2, 5, (byte) 0x2A,
+                                                 1, 0, 0, 0,
+                                             }, extra);
+                                     } finally {
+                                         a.close();
+                                     }
+                                 }
+                             },
+                             false);
+    }
+
     static interface ZipOutputTest {
         void test(File f, ZipArchiveOutputStream zos) throws IOException;
     }



Mime
View raw message