Author: ebourg
Date: Mon Dec 16 13:18:24 2013
New Revision: 1551202
URL: http://svn.apache.org/r1551202
Log:
GzipCompressorOutputStream revamp to support custom compression level and
header metadata (COMPRESS-250)
Added:
commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/gzip/GzipParameters.java
(with props)
Modified:
commons/proper/compress/trunk/src/changes/changes.xml
commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/gzip/GzipCompressorOutputStream.java
commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/GZipTestCase.java
Modified: commons/proper/compress/trunk/src/changes/changes.xml
URL:
http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/changes/changes.xml?rev=1551202&r1=1551201&r2=1551202&view=diff
==============================================================================
--- commons/proper/compress/trunk/src/changes/changes.xml (original)
+++ commons/proper/compress/trunk/src/changes/changes.xml Mon Dec 16 13:18:24
2013
@@ -44,6 +44,10 @@ The <action> type attribute can be add,u
<body>
<release version="1.7" date="not released, yet"
description="Release 1.7">
+ <action issue="COMPRESS-250" type="add" date="2012-12-16"
due-to="Emmanuel Bourg">
+ GzipCompressorOutputStream now supports setting the compression level
and the header metadata
+ (filename, comment, modification time, operating system and extra
flags)
+ </action>
<action issue="COMPRESS-241" type="fix" date="2013-10-27">
SevenZOutputFile#closeArchiveEntry throws an exception when
using LZMA2 compression on Java8.
Modified:
commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/gzip/GzipCompressorOutputStream.java
URL:
http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/gzip/GzipCompressorOutputStream.java?rev=1551202&r1=1551201&r2=1551202&view=diff
==============================================================================
---
commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/gzip/GzipCompressorOutputStream.java
(original)
+++
commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/gzip/GzipCompressorOutputStream.java
Mon Dec 16 13:18:24 2013
@@ -20,21 +20,113 @@ package org.apache.commons.compress.comp
import java.io.IOException;
import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.zip.CRC32;
+import java.util.zip.Deflater;
+import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import org.apache.commons.compress.compressors.CompressorOutputStream;
+/**
+ * Compressed output stream using the gzip format. This implementation improves
+ * over the standard {@link GZIPOutputStream} class by allowing
+ * the configuration of the compression level and the header metadata
(filename,
+ * comment, modification time, operating system and extra flags).
+ *
+ * @see <a href="http://tools.ietf.org/html/rfc1952";>GZIP File Format
Specification</a>
+ */
public class GzipCompressorOutputStream extends CompressorOutputStream {
- private final GZIPOutputStream out;
+ /** Header flag indicating a file name follows the header */
+ private static final int FNAME = 1 << 3;
- public GzipCompressorOutputStream( final OutputStream outputStream )
throws IOException {
- out = new GZIPOutputStream(outputStream);
+ /** Header flag indicating a comment follows the header */
+ private static final int FCOMMENT = 1 << 4;
+
+ /** The underlying stream */
+ private final OutputStream out;
+
+ /** Deflater used to compress the data */
+ private final Deflater deflater;
+
+ /** The buffer receiving the compressed data from the deflater */
+ private final byte[] buffer = new byte[512];
+
+ /** Indicates if the stream has been closed */
+ private boolean closed;
+
+ /** The checksum of the uncompressed data */
+ private final CRC32 crc = new CRC32();
+
+ /**
+ * Creates a gzip compressed output stream with the default parameters.
+ */
+ public GzipCompressorOutputStream(OutputStream out) throws IOException {
+ this(out, new GzipParameters());
+ }
+
+ /**
+ * Creates a gzip compressed output stream with the specified parameters.
+ *
+ * @since 1.7
+ */
+ public GzipCompressorOutputStream(OutputStream out, GzipParameters
parameters) throws IOException {
+ this.out = out;
+ this.deflater = new Deflater(parameters.getCompressionLevel(), true);
+
+ writeHeader(parameters);
+ }
+
+ private void writeHeader(GzipParameters parameters) throws IOException {
+ String filename = parameters.getFilename();
+ String comment = parameters.getComment();
+
+ ByteBuffer buffer = ByteBuffer.allocate(10);
+ buffer.order(ByteOrder.LITTLE_ENDIAN);
+ buffer.putShort((short) GZIPInputStream.GZIP_MAGIC);
+ buffer.put((byte) 8); // compression method (8: deflate)
+ buffer.put((byte) ((filename != null ? FNAME : 0) | (comment != null ?
FCOMMENT : 0))); // flags
+ buffer.putInt((int) (parameters.getModificationTime() / 1000));
+
+ // extra flags
+ int compressionLevel = parameters.getCompressionLevel();
+ if (compressionLevel == Deflater.BEST_COMPRESSION) {
+ buffer.put((byte) 2);
+ } else if (compressionLevel == Deflater.BEST_SPEED) {
+ buffer.put((byte) 4);
+ } else {
+ buffer.put((byte) 0);
+ }
+
+ buffer.put((byte) parameters.getOperatingSystem());
+
+ out.write(buffer.array());
+
+ if (filename != null) {
+ out.write(filename.getBytes("ISO-8859-1"));
+ out.write(0);
+ }
+
+ if (comment != null) {
+ out.write(comment.getBytes("ISO-8859-1"));
+ out.write(0);
+ }
+ }
+
+ private void writeTrailer() throws IOException {
+ ByteBuffer buffer = ByteBuffer.allocate(8);
+ buffer.order(ByteOrder.LITTLE_ENDIAN);
+ buffer.putInt((int) crc.getValue());
+ buffer.putInt(deflater.getTotalIn());
+
+ out.write(buffer.array());
}
@Override
public void write(int b) throws IOException {
- out.write(b);
+ write(new byte[]{(byte) (b & 0xff)}, 0, 1);
}
/**
@@ -43,8 +135,8 @@ public class GzipCompressorOutputStream
* @since 1.1
*/
@Override
- public void write(byte[] b) throws IOException {
- out.write(b);
+ public void write(byte[] buffer) throws IOException {
+ write(buffer, 0, buffer.length);
}
/**
@@ -53,8 +145,26 @@ public class GzipCompressorOutputStream
* @since 1.1
*/
@Override
- public void write(byte[] b, int from, int length) throws IOException {
- out.write(b, from, length);
+ public void write(byte[] buffer, int offset, int length) throws
IOException {
+ if (deflater.finished()) {
+ throw new IOException("Cannot write more data, the end of the
compressed data stream has been reached");
+
+ } else if (length > 0) {
+ deflater.setInput(buffer, offset, length);
+
+ while (!deflater.needsInput()) {
+ deflate();
+ }
+
+ crc.update(buffer, offset, length);
+ }
+ }
+
+ private void deflate() throws IOException {
+ int length = deflater.deflate(buffer, 0, buffer.length);
+ if (length > 0) {
+ out.write(buffer, 0, length);
+ }
}
/**
@@ -63,7 +173,15 @@ public class GzipCompressorOutputStream
* @since 1.7
*/
public void finish() throws IOException {
- out.finish();
+ if (!deflater.finished()) {
+ deflater.finish();
+
+ while (!deflater.finished()) {
+ deflate();
+ }
+
+ writeTrailer();
+ }
}
/**
@@ -78,7 +196,12 @@ public class GzipCompressorOutputStream
@Override
public void close() throws IOException {
- out.close();
+ if (!closed) {
+ finish();
+ deflater.end();
+ out.close();
+ closed = true;
+ }
}
}
Added:
commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/gzip/GzipParameters.java
URL:
http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/gzip/GzipParameters.java?rev=1551202&view=auto
==============================================================================
---
commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/gzip/GzipParameters.java
(added)
+++
commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/gzip/GzipParameters.java
Mon Dec 16 13:18:24 2013
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.commons.compress.compressors.gzip;
+
+import java.util.zip.Deflater;
+
+/**
+ * Parameters for the GZIP compressor.
+ *
+ * @since 1.7
+ */
+public class GzipParameters {
+
+ private int compressionLevel = Deflater.DEFAULT_COMPRESSION;
+ private long modificationTime;
+ private String filename;
+ private String comment;
+ private int operatingSystem = 255; // Unknown OS by default
+
+ public int getCompressionLevel() {
+ return compressionLevel;
+ }
+
+ /**
+ * Sets the compression level.
+ *
+ * @param compressionLevel the compression level (between 0 and 9)
+ * @see Deflater#NO_COMPRESSION
+ * @see Deflater#BEST_SPEED
+ * @see Deflater#DEFAULT_COMPRESSION
+ * @see Deflater#BEST_COMPRESSION
+ */
+ public void setCompressionLevel(int compressionLevel) {
+ if (compressionLevel < -1 || compressionLevel > 9) {
+ throw new IllegalArgumentException("Invalid gzip compression
level: " + compressionLevel);
+ }
+ this.compressionLevel = compressionLevel;
+ }
+
+ public long getModificationTime() {
+ return modificationTime;
+ }
+
+ /**
+ * Sets the modification time of the compressed file.
+ *
+ * @param modificationTime the modification time, in milliseconds
+ */
+ public void setModificationTime(long modificationTime) {
+ this.modificationTime = modificationTime;
+ }
+
+ public String getFilename() {
+ return filename;
+ }
+
+ /**
+ * Sets the name of the compressed file.
+ *
+ * @param filename the name of the file without the directory path
+ */
+ public void setFilename(String filename) {
+ this.filename = filename;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ public int getOperatingSystem() {
+ return operatingSystem;
+ }
+
+ /**
+ * Sets the operating system on which the compression took place.
+ * The defined values are:
+ * <ul>
+ * <li>0: FAT filesystem (MS-DOS, OS/2, NT/Win32)</li>
+ * <li>1: Amiga</li>
+ * <li>2: VMS (or OpenVMS)</li>
+ * <li>3: Unix</li>
+ * <li>4: VM/CMS</li>
+ * <li>5: Atari TOS</li>
+ * <li>6: HPFS filesystem (OS/2, NT)</li>
+ * <li>7: Macintosh</li>
+ * <li>8: Z-System</li>
+ * <li>9: CP/M</li>
+ * <li>10: TOPS-20</li>
+ * <li>11: NTFS filesystem (NT)</li>
+ * <li>12: QDOS</li>
+ * <li>13: Acorn RISCOS</li>
+ * <li>255: Unknown</li>
+ * </ul>
+ *
+ * @param operatingSystem the code of the operating system
+ */
+ public void setOperatingSystem(int operatingSystem) {
+ this.operatingSystem = operatingSystem;
+ }
+}
Propchange:
commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/gzip/GzipParameters.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/compressors/gzip/GzipParameters.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Modified:
commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/GZipTestCase.java
URL:
http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/GZipTestCase.java?rev=1551202&r1=1551201&r2=1551202&view=diff
==============================================================================
---
commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/GZipTestCase.java
(original)
+++
commons/proper/compress/trunk/src/test/java/org/apache/commons/compress/compressors/GZipTestCase.java
Mon Dec 16 13:18:24 2013
@@ -26,10 +26,17 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.util.zip.Deflater;
+import java.util.zip.GZIPInputStream;
import org.apache.commons.compress.AbstractTestCase;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipParameters;
import org.apache.commons.compress.utils.IOUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.output.NullOutputStream;
+import org.junit.Assert;
public final class GZipTestCase extends AbstractTestCase {
@@ -147,4 +154,103 @@ public final class GZipTestCase extends
}
}
}
+
+ public void testInteroperabilityWithGzipCompressorInputStream() throws
Exception {
+ byte[] content = FileUtils.readFileToByteArray(getFile("test3.xml"));
+
+ ByteArrayOutputStream bout = new ByteArrayOutputStream();
+
+ GzipParameters parameters = new GzipParameters();
+ parameters.setCompressionLevel(Deflater.BEST_COMPRESSION);
+ parameters.setOperatingSystem(3);
+ parameters.setFilename("test3.xml");
+ parameters.setComment("Test file");
+ parameters.setModificationTime(System.currentTimeMillis());
+ GzipCompressorOutputStream out = new GzipCompressorOutputStream(bout,
parameters);
+ out.write(content);
+ out.flush();
+ out.close();
+
+ GzipCompressorInputStream in = new GzipCompressorInputStream(new
ByteArrayInputStream(bout.toByteArray()));
+ byte[] content2 = IOUtils.toByteArray(in);
+
+ Assert.assertArrayEquals("uncompressed content", content, content2);
+ }
+
+ public void testInteroperabilityWithGZIPInputStream() throws Exception {
+ byte[] content = FileUtils.readFileToByteArray(getFile("test3.xml"));
+
+ ByteArrayOutputStream bout = new ByteArrayOutputStream();
+
+ GzipParameters parameters = new GzipParameters();
+ parameters.setCompressionLevel(Deflater.BEST_COMPRESSION);
+ parameters.setOperatingSystem(3);
+ parameters.setFilename("test3.xml");
+ parameters.setComment("Test file");
+ parameters.setModificationTime(System.currentTimeMillis());
+ GzipCompressorOutputStream out = new GzipCompressorOutputStream(bout,
parameters);
+ out.write(content);
+ out.flush();
+ out.close();
+
+ GZIPInputStream in = new GZIPInputStream(new
ByteArrayInputStream(bout.toByteArray()));
+ byte[] content2 = IOUtils.toByteArray(in);
+
+ Assert.assertArrayEquals("uncompressed content", content, content2);
+ }
+
+ public void testInvalidCompressionLevel() {
+ GzipParameters parameters = new GzipParameters();
+ try {
+ parameters.setCompressionLevel(10);
+ fail("IllegalArgumentException not thrown");
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+
+ try {
+ parameters.setCompressionLevel(-5);
+ fail("IllegalArgumentException not thrown");
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+ }
+
+ private void testExtraFlags(int compressionLevel, int flag) throws
Exception {
+ byte[] content = FileUtils.readFileToByteArray(getFile("test3.xml"));
+
+ ByteArrayOutputStream bout = new ByteArrayOutputStream();
+
+ GzipParameters parameters = new GzipParameters();
+ parameters.setCompressionLevel(compressionLevel);
+ GzipCompressorOutputStream out = new GzipCompressorOutputStream(bout,
parameters);
+ IOUtils.copy(new ByteArrayInputStream(content), out);
+ out.flush();
+ out.close();
+
+ assertEquals("extra flags (XFL)", flag, bout.toByteArray()[8]);
+ }
+
+ public void testExtraFlagsFastestCompression() throws Exception {
+ testExtraFlags(Deflater.BEST_SPEED, 4);
+ }
+
+ public void testExtraFlagsBestCompression() throws Exception {
+ testExtraFlags(Deflater.BEST_COMPRESSION, 2);
+ }
+
+ public void testExtraFlagsDefaultCompression() throws Exception {
+ testExtraFlags(Deflater.DEFAULT_COMPRESSION, 0);
+ }
+
+ public void testOverWrite() throws Exception {
+ GzipCompressorOutputStream out = new GzipCompressorOutputStream(new
NullOutputStream());
+ out.close();
+ try {
+ out.write(0);
+ fail("IOException expected");
+ } catch (IOException e) {
+ // expected
+ }
+ }
}
