Return-Path: X-Original-To: apmail-commons-commits-archive@minotaur.apache.org Delivered-To: apmail-commons-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id DF8159CC2 for ; Thu, 8 Dec 2011 15:22:42 +0000 (UTC) Received: (qmail 87469 invoked by uid 500); 8 Dec 2011 15:22:42 -0000 Delivered-To: apmail-commons-commits-archive@commons.apache.org Received: (qmail 87413 invoked by uid 500); 8 Dec 2011 15:22:42 -0000 Mailing-List: contact commits-help@commons.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@commons.apache.org Delivered-To: mailing list commits@commons.apache.org Received: (qmail 87406 invoked by uid 99); 8 Dec 2011 15:22:42 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 08 Dec 2011 15:22:42 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 08 Dec 2011 15:22:39 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 450FF23889DA for ; Thu, 8 Dec 2011 15:22:17 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1211931 - in /commons/proper/compress/trunk/src: main/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStream.java test/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStreamTest.java Date: Thu, 08 Dec 2011 15:22:17 -0000 To: commits@commons.apache.org From: bodewig@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20111208152217.450FF23889DA@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: bodewig Date: Thu Dec 8 15:22:16 2011 New Revision: 1211931 URL: http://svn.apache.org/viewvc?rev=1211931&view=rev Log: Write big files in PAX/POSIX mode. COMPRESS-165 Modified: commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStream.java commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStreamTest.java Modified: commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStream.java URL: http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStream.java?rev=1211931&r1=1211930&r2=1211931&view=diff ============================================================================== --- commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStream.java (original) +++ commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStream.java Thu Dec 8 15:22:16 2011 @@ -21,6 +21,9 @@ package org.apache.commons.compress.arch import java.io.File; import java.io.IOException; import java.io.OutputStream; +import java.io.StringWriter; +import java.util.HashMap; +import java.util.Map; import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.utils.ArchiveUtils; @@ -207,6 +210,7 @@ public class TarArchiveOutputStream exte throw new IOException("Stream has already been finished"); } TarArchiveEntry entry = (TarArchiveEntry) archiveEntry; + Map paxHeaders = new HashMap(); if (entry.getName().length() >= TarConstants.NAMELEN) { if (longFileMode == LONGFILE_GNU) { @@ -227,14 +231,21 @@ public class TarArchiveOutputStream exte + TarConstants.NAMELEN + " bytes)"); } } + if (entry.getSize() > TarConstants.MAXSIZE) { - if (bigFileMode != BIGFILE_STAR) { + if (bigFileMode == BIGFILE_POSIX) { + paxHeaders.put("size", String.valueOf(entry.getSize())); + } else if (bigFileMode != BIGFILE_STAR) { throw new RuntimeException("file size '" + entry.getSize() + "' is too big ( > " + TarConstants.MAXSIZE + " bytes)"); } } + if (paxHeaders.size() > 0) { + writePaxHeaders(entry.getName(), paxHeaders); + } + entry.writeEntryHeader(recordBuf, bigFileMode == BIGFILE_STAR); buffer.writeRecord(recordBuf); @@ -368,6 +379,47 @@ public class TarArchiveOutputStream exte } /** + * Writes a PAX extended header with the given map as contents. + * @since Apache Commons Compress 1.4 + */ + void writePaxHeaders(String entryName, + Map headers) throws IOException { + String name = "./PaxHeaders.X/" + entryName; + if (name.length() > TarConstants.NAMELEN) { + name = name.substring(0, TarConstants.NAMELEN); + } + TarArchiveEntry pex = new TarArchiveEntry(name, + TarConstants.LF_PAX_EXTENDED_HEADER_LC); + + StringWriter w = new StringWriter(); + for (Map.Entry h : headers.entrySet()) { + String key = h.getKey(); + String value = h.getValue(); + int len = key.length() + value.length() + + 3 /* blank, equals and newline */ + + 2 /* guess 9 < actual length < 100 */; + String line = len + " " + key + "=" + value + "\n"; + int actualLength = line.getBytes("UTF-8").length; + while (len != actualLength) { + // Adjust for cases where length < 10 or > 100 + // or where UTF-8 encoding isn't a single octet + // per character. + // Must be in loop as size may go from 99 to 100 in + // first pass so we'd need a second. + len = actualLength; + line = len + " " + key + "=" + value + "\n"; + actualLength = line.getBytes("UTF-8").length; + } + w.write(line); + } + byte[] data = w.toString().getBytes("UTF-8"); + pex.setSize(data.length); + putArchiveEntry(pex); + write(data); + closeArchiveEntry(); + } + + /** * Write an EOF (end of archive) record to the tar archive. * An EOF record consists of a record of all zeros. */ Modified: commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStreamTest.java URL: http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStreamTest.java?rev=1211931&r1=1211930&r2=1211931&view=diff ============================================================================== --- commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStreamTest.java (original) +++ commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStreamTest.java Thu Dec 8 15:22:16 2011 @@ -23,6 +23,8 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.util.HashMap; +import java.util.Map; import org.apache.commons.compress.AbstractTestCase; import org.apache.commons.compress.archivers.ArchiveOutputStream; @@ -93,4 +95,91 @@ public class TarArchiveOutputStreamTest TarArchiveEntry e = tin.getNextTarEntry(); assertEquals(0100000000000L, e.getSize()); } + + public void testBigFilePosixMode() throws Exception { + TarArchiveEntry t = new TarArchiveEntry("foo"); + t.setSize(0100000000000L); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + TarArchiveOutputStream tos = new TarArchiveOutputStream(bos); + tos.setBigFileMode(TarArchiveOutputStream.BIGFILE_POSIX); + tos.putArchiveEntry(t); + // make sure header is written to byte array + tos.write(new byte[10 * 1024]); + byte[] data = bos.toByteArray(); + assertEquals("00000000000 ", + new String(data, + 1024 + TarConstants.NAMELEN + + TarConstants.MODELEN + + TarConstants.UIDLEN + + TarConstants.GIDLEN, 12, + "UTF-8")); + TarArchiveInputStream tin = + new TarArchiveInputStream(new ByteArrayInputStream(data)); + TarArchiveEntry e = tin.getNextTarEntry(); + assertEquals(0100000000000L, e.getSize()); + } + + public void testWriteSimplePaxHeaders() throws Exception { + Map m = new HashMap(); + m.put("a", "b"); + byte[] data = writePaxHeader(m); + assertEquals("00000000006 ", + new String(data, TarConstants.NAMELEN + + TarConstants.MODELEN + + TarConstants.UIDLEN + + TarConstants.GIDLEN, 12, + "UTF-8")); + assertEquals("6 a=b\n", new String(data, 512, 6, "UTF-8")); + } + + public void testPaxHeadersWithLength99() throws Exception { + Map m = new HashMap(); + m.put("a", + "0123456789012345678901234567890123456789" + + "01234567890123456789012345678901234567890123456789" + + "012"); + byte[] data = writePaxHeader(m); + assertEquals("00000000143 ", + new String(data, TarConstants.NAMELEN + + TarConstants.MODELEN + + TarConstants.UIDLEN + + TarConstants.GIDLEN, 12, + "UTF-8")); + assertEquals("99 a=0123456789012345678901234567890123456789" + + "01234567890123456789012345678901234567890123456789" + + "012\n", new String(data, 512, 99, "UTF-8")); + } + + public void testPaxHeadersWithLength101() throws Exception { + Map m = new HashMap(); + m.put("a", + "0123456789012345678901234567890123456789" + + "01234567890123456789012345678901234567890123456789" + + "0123"); + byte[] data = writePaxHeader(m); + assertEquals("00000000145 ", + new String(data, TarConstants.NAMELEN + + TarConstants.MODELEN + + TarConstants.UIDLEN + + TarConstants.GIDLEN, 12, + "UTF-8")); + assertEquals("101 a=0123456789012345678901234567890123456789" + + "01234567890123456789012345678901234567890123456789" + + "0123\n", new String(data, 512, 101, "UTF-8")); + } + + private byte[] writePaxHeader(Map m) throws Exception { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + TarArchiveOutputStream tos = new TarArchiveOutputStream(bos); + tos.writePaxHeaders("foo", m); + + // add a dummy entry so data gets written + TarArchiveEntry t = new TarArchiveEntry("foo"); + t.setSize(10 * 1024); + tos.putArchiveEntry(t); + tos.write(new byte[10 * 1024]); + tos.closeArchiveEntry(); + + return bos.toByteArray(); + } } \ No newline at end of file