Author: bodewig Date: Thu Dec 8 15:22:16 2011 New Revision: 1211931 URL: http://svn.apache.org/viewvc?rev=1211931&view=rev Log: Write big files in PAX/POSIX mode. COMPRESS-165
Modified: commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStream.java commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStreamTest.java Modified: commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStream.java URL: http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStream.java?rev=1211931&r1=1211930&r2=1211931&view=diff ============================================================================== --- commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStream.java (original) +++ commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStream.java Thu Dec 8 15:22:16 2011 @@ -21,6 +21,9 @@ package org.apache.commons.compress.arch import java.io.File; import java.io.IOException; import java.io.OutputStream; +import java.io.StringWriter; +import java.util.HashMap; +import java.util.Map; import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.utils.ArchiveUtils; @@ -207,6 +210,7 @@ public class TarArchiveOutputStream exte throw new IOException("Stream has already been finished"); } TarArchiveEntry entry = (TarArchiveEntry) archiveEntry; + Map<String, String> paxHeaders = new HashMap<String, String>(); if (entry.getName().length() >= TarConstants.NAMELEN) { if (longFileMode == LONGFILE_GNU) { @@ -227,14 +231,21 @@ public class TarArchiveOutputStream exte + TarConstants.NAMELEN + " bytes)"); } } + if (entry.getSize() > TarConstants.MAXSIZE) { - if (bigFileMode != BIGFILE_STAR) { + if (bigFileMode == BIGFILE_POSIX) { + paxHeaders.put("size", String.valueOf(entry.getSize())); + } else if (bigFileMode != BIGFILE_STAR) { throw new RuntimeException("file size '" + entry.getSize() + "' is too big ( > " + TarConstants.MAXSIZE + " bytes)"); } } + if (paxHeaders.size() > 0) { + writePaxHeaders(entry.getName(), paxHeaders); + } + entry.writeEntryHeader(recordBuf, bigFileMode == BIGFILE_STAR); buffer.writeRecord(recordBuf); @@ -368,6 +379,47 @@ public class TarArchiveOutputStream exte } /** + * Writes a PAX extended header with the given map as contents. + * @since Apache Commons Compress 1.4 + */ + void writePaxHeaders(String entryName, + Map<String, String> headers) throws IOException { + String name = "./PaxHeaders.X/" + entryName; + if (name.length() > TarConstants.NAMELEN) { + name = name.substring(0, TarConstants.NAMELEN); + } + TarArchiveEntry pex = new TarArchiveEntry(name, + TarConstants.LF_PAX_EXTENDED_HEADER_LC); + + StringWriter w = new StringWriter(); + for (Map.Entry<String, String> h : headers.entrySet()) { + String key = h.getKey(); + String value = h.getValue(); + int len = key.length() + value.length() + + 3 /* blank, equals and newline */ + + 2 /* guess 9 < actual length < 100 */; + String line = len + " " + key + "=" + value + "\n"; + int actualLength = line.getBytes("UTF-8").length; + while (len != actualLength) { + // Adjust for cases where length < 10 or > 100 + // or where UTF-8 encoding isn't a single octet + // per character. + // Must be in loop as size may go from 99 to 100 in + // first pass so we'd need a second. + len = actualLength; + line = len + " " + key + "=" + value + "\n"; + actualLength = line.getBytes("UTF-8").length; + } + w.write(line); + } + byte[] data = w.toString().getBytes("UTF-8"); + pex.setSize(data.length); + putArchiveEntry(pex); + write(data); + closeArchiveEntry(); + } + + /** * Write an EOF (end of archive) record to the tar archive. * An EOF record consists of a record of all zeros. */ Modified: commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStreamTest.java URL: http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStreamTest.java?rev=1211931&r1=1211930&r2=1211931&view=diff ============================================================================== --- commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStreamTest.java (original) +++ commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/archivers/tar/TarArchiveOutputStreamTest.java Thu Dec 8 15:22:16 2011 @@ -23,6 +23,8 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.util.HashMap; +import java.util.Map; import org.apache.commons.compress.AbstractTestCase; import org.apache.commons.compress.archivers.ArchiveOutputStream; @@ -93,4 +95,91 @@ public class TarArchiveOutputStreamTest TarArchiveEntry e = tin.getNextTarEntry(); assertEquals(0100000000000L, e.getSize()); } + + public void testBigFilePosixMode() throws Exception { + TarArchiveEntry t = new TarArchiveEntry("foo"); + t.setSize(0100000000000L); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + TarArchiveOutputStream tos = new TarArchiveOutputStream(bos); + tos.setBigFileMode(TarArchiveOutputStream.BIGFILE_POSIX); + tos.putArchiveEntry(t); + // make sure header is written to byte array + tos.write(new byte[10 * 1024]); + byte[] data = bos.toByteArray(); + assertEquals("00000000000 ", + new String(data, + 1024 + TarConstants.NAMELEN + + TarConstants.MODELEN + + TarConstants.UIDLEN + + TarConstants.GIDLEN, 12, + "UTF-8")); + TarArchiveInputStream tin = + new TarArchiveInputStream(new ByteArrayInputStream(data)); + TarArchiveEntry e = tin.getNextTarEntry(); + assertEquals(0100000000000L, e.getSize()); + } + + public void testWriteSimplePaxHeaders() throws Exception { + Map<String, String> m = new HashMap<String, String>(); + m.put("a", "b"); + byte[] data = writePaxHeader(m); + assertEquals("00000000006 ", + new String(data, TarConstants.NAMELEN + + TarConstants.MODELEN + + TarConstants.UIDLEN + + TarConstants.GIDLEN, 12, + "UTF-8")); + assertEquals("6 a=b\n", new String(data, 512, 6, "UTF-8")); + } + + public void testPaxHeadersWithLength99() throws Exception { + Map<String, String> m = new HashMap<String, String>(); + m.put("a", + "0123456789012345678901234567890123456789" + + "01234567890123456789012345678901234567890123456789" + + "012"); + byte[] data = writePaxHeader(m); + assertEquals("00000000143 ", + new String(data, TarConstants.NAMELEN + + TarConstants.MODELEN + + TarConstants.UIDLEN + + TarConstants.GIDLEN, 12, + "UTF-8")); + assertEquals("99 a=0123456789012345678901234567890123456789" + + "01234567890123456789012345678901234567890123456789" + + "012\n", new String(data, 512, 99, "UTF-8")); + } + + public void testPaxHeadersWithLength101() throws Exception { + Map<String, String> m = new HashMap<String, String>(); + m.put("a", + "0123456789012345678901234567890123456789" + + "01234567890123456789012345678901234567890123456789" + + "0123"); + byte[] data = writePaxHeader(m); + assertEquals("00000000145 ", + new String(data, TarConstants.NAMELEN + + TarConstants.MODELEN + + TarConstants.UIDLEN + + TarConstants.GIDLEN, 12, + "UTF-8")); + assertEquals("101 a=0123456789012345678901234567890123456789" + + "01234567890123456789012345678901234567890123456789" + + "0123\n", new String(data, 512, 101, "UTF-8")); + } + + private byte[] writePaxHeader(Map<String, String> m) throws Exception { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + TarArchiveOutputStream tos = new TarArchiveOutputStream(bos); + tos.writePaxHeaders("foo", m); + + // add a dummy entry so data gets written + TarArchiveEntry t = new TarArchiveEntry("foo"); + t.setSize(10 * 1024); + tos.putArchiveEntry(t); + tos.write(new byte[10 * 1024]); + tos.closeArchiveEntry(); + + return bos.toByteArray(); + } } \ No newline at end of file