There are several places where single byte writes are being done during compression. Often this is going to an OutputStream with synchronized write methods. Historically that has not mattered much because of biased locking. However, biased locking is being removed[1]. These changes will batch those writes up to a small buffer.
[1] - https://openjdk.java.net/jeps/374 diff --git a/src/org/tukaani/xz/LZMA2OutputStream.java b/src/org/tukaani/xz/LZMA2OutputStream.java index a82a1a5..14bb3a9 100644xz/LZMA2OutputStream.java b/src/org/tukaani/xz/LZM --- a/src/org/tukaani/xz/LZMA2OutputStream.java +++ b/src/org/tukaani/xz/LZMA2OutputStream.java @@ -10,19 +10,19 @@ package org.tukaani.xz; -import java.io.DataOutputStream; import java.io.IOException; + import org.tukaani.xz.lz.LZEncoder; -import org.tukaani.xz.rangecoder.RangeEncoderToBuffer; import org.tukaani.xz.lzma.LZMAEncoder; +import org.tukaani.xz.rangecoder.RangeEncoderToBuffer; class LZMA2OutputStream extends FinishableOutputStream { static final int COMPRESSED_SIZE_MAX = 64 << 10; private final ArrayCache arrayCache; + private final byte[] buffer = new byte[8]; private FinishableOutputStream out; - private final DataOutputStream outData; private LZEncoder lz; private RangeEncoderToBuffer rc; @@ -60,7 +60,6 @@ class LZMA2OutputStream extends FinishableOutputStream { this.arrayCache = arrayCache; this.out = out; - outData = new DataOutputStream(out); rc = new RangeEncoderToBuffer(COMPRESSED_SIZE_MAX, arrayCache); int dictSize = options.getDictSize(); @@ -154,13 +153,17 @@ class LZMA2OutputStream extends FinishableOutputStream { } control |= (uncompressedSize - 1) >>> 16; - outData.writeByte(control); + buffer[0] = (byte) control; - outData.writeShort(uncompressedSize - 1); - outData.writeShort(compressedSize - 1); + writeBEShort(uncompressedSize - 1, buffer, 1); + writeBEShort(compressedSize - 1, buffer, 3); - if (propsNeeded) - outData.writeByte(props); + int length = 5; + if (propsNeeded) { + buffer[5] = (byte) props; + length = 6; + } + out.write(buffer, 0, length); rc.write(out); @@ -172,8 +175,9 @@ class LZMA2OutputStream extends FinishableOutputStream { private void writeUncompressed(int uncompressedSize) throws IOException { while (uncompressedSize > 0) { int chunkSize = Math.min(uncompressedSize, COMPRESSED_SIZE_MAX); - outData.writeByte(dictResetNeeded ? 0x01 : 0x02); - outData.writeShort(chunkSize - 1); + buffer[0] = (byte) (dictResetNeeded ? 0x01 : 0x02); + writeBEShort(chunkSize - 1, buffer, 1); + out.write(buffer, 0, 3); lz.copyUncompressed(out, uncompressedSize, chunkSize); uncompressedSize -= chunkSize; dictResetNeeded = false; @@ -182,6 +186,15 @@ class LZMA2OutputStream extends FinishableOutputStream { stateResetNeeded = true; } + /** + * Writes the right-most 2 bytes of <i>s</i> to <i>b</i>, starting at + * <i>off</i> with the most significant byte first (BigEndian). + */ + private static void writeBEShort(int s, byte[] b, int off) { + b[off] = (byte) ((s >>> 8) & 0xFF); + b[off + 1] = (byte) (s & 0xFF); + } + private void writeEndMarker() throws IOException { assert !finished; diff --git a/src/org/tukaani/xz/common/EncoderUtil.java b/src/org/tukaani/xz/common/EncoderUtil.java index 57f688b..bfb2f78 100644 --- a/src/org/tukaani/xz/common/EncoderUtil.java +++ b/src/org/tukaani/xz/common/EncoderUtil.java @@ -9,21 +9,80 @@ package org.tukaani.xz.common; -import java.io.OutputStream; import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.zip.CRC32; public class EncoderUtil extends Util { + /** + * @deprecated Prefer use of either {@link #addCRC32(byte[], int, int, int)} + * or {@link #appendCRC32(ByteBuffer)}. + */ + @Deprecated public static void writeCRC32(OutputStream out, byte[] buf) throws IOException { CRC32 crc32 = new CRC32(); crc32.update(buf); long value = crc32.getValue(); - for (int i = 0; i < 4; ++i) - out.write((byte)(value >>> (i * 8))); + out.write((byte) value); + out.write((byte)(value >>> 8)); + out.write((byte)(value >>> 16)); + out.write((byte)(value >>> 24)); + } + + /** + * Calculates the crc32 for <i>length</i> bytes in <i>buf</i> starting at + * <i>off</i> and inserts the crc32 value as a little endian {@code int} + * at <i>crcOff</i>. + * @param buf The buffer to calculate crc32 for and insert value into. + * @param off The offset into <i>buf</i> to start reading for crc32 calc. + * @param length The number of bytes to include in crc32 calc. + * @param crcOff The offset into <i>buf</i> to write the crc32 value to. + * @throws IOException + */ + public static void calcAndInsertCRC32(byte[] buf, int off, int length, int crcOff) + throws IOException { + CRC32 crc32 = new CRC32(); + crc32.update(buf, off, length); + long value = crc32.getValue(); + + buf[crcOff] = (byte) value; + buf[crcOff + 1] = (byte)(value >>> 8); + buf[crcOff + 2] = (byte)(value >>> 16); + buf[crcOff + 3] = (byte)(value >>> 24); + } + + /** + * Calculates the crc32 for all {@link ByteBuffer#remaining()} bytes, + * then extends the {@link ByteBuffer#limit()} by {@code 4} and appends + * the crc32 value as a little endian {@code int}. + * @param bb The buffer with contents to calculate crc32 and append that + * value to. + * @throws IOException + */ + public static void appendCRC32(ByteBuffer bb) + throws IOException { + CRC32 crc32 = new CRC32(); + if (bb.hasArray()) { + crc32.update(bb.array(), bb.arrayOffset() + bb.position(), bb.remaining()); + } else { + throw new UnsupportedOperationException(); + } + long value = crc32.getValue(); + + bb.order(ByteOrder.LITTLE_ENDIAN); + bb.position(bb.limit()); + bb.limit(bb.limit() + 4); + bb.putInt((int) value); } + /** + * @deprecated Prefer use of {@link #encodeVLI(OutputStream, long)}. + */ + @Deprecated public static void encodeVLI(OutputStream out, long num) throws IOException { while (num >= 0x80) { @@ -33,4 +92,20 @@ public class EncoderUtil extends Util { out.write((byte)num); } + + /** + * Appends the vli representation of <i>num</i> to <i>bb</i>. + * @param bb The buffer to encode the <i>num</i> to. + * @param num The number to encode. + * @throws IOException + */ + public static void encodeVLI(ByteBuffer bb, long num) + throws IOException { + while (num >= 0x80) { + bb.put((byte)(num | 0x80)); + num >>>= 7; + } + + bb.put((byte)num); + } } diff --git a/src/org/tukaani/xz/BlockOutputStream.java b/src/org/tukaani/xz/BlockOutputStream.java index 8ac4407..3e2493b 100644 --- a/src/org/tukaani/xz/BlockOutputStream.java +++ b/src/org/tukaani/xz/BlockOutputStream.java @@ -9,11 +9,12 @@ package org.tukaani.xz; -import java.io.OutputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import org.tukaani.xz.common.EncoderUtil; +import java.io.OutputStream; +import java.nio.ByteBuffer; + import org.tukaani.xz.check.Check; +import org.tukaani.xz.common.EncoderUtil; class BlockOutputStream extends FinishableOutputStream { private final OutputStream out; @@ -40,44 +41,57 @@ class BlockOutputStream extends FinishableOutputStream { filterChain = filters[i].getOutputStream(filterChain, arrayCache); // Prepare to encode the Block Header field. - ByteArrayOutputStream bufStream = new ByteArrayOutputStream(); + ByteBuffer blockBB = ByteBuffer.allocate(32); - // Write a dummy Block Header Size field. The real value is written + // Skip the Block Header Size field. The real value is written // once everything else except CRC32 has been written. - bufStream.write(0x00); + blockBB.position(1); // Write Block Flags. Storing Compressed Size or Uncompressed Size // isn't supported for now. - bufStream.write(filters.length - 1); + blockBB.put((byte) (filters.length - 1)); // List of Filter Flags for (int i = 0; i < filters.length; ++i) { - EncoderUtil.encodeVLI(bufStream, filters[i].getFilterID()); + EncoderUtil.encodeVLI(blockBB, filters[i].getFilterID()); byte[] filterProps = filters[i].getFilterProps(); - EncoderUtil.encodeVLI(bufStream, filterProps.length); - bufStream.write(filterProps); + if (filterProps.length + 16 > blockBB.remaining()) { + blockBB = resize(blockBB); + } + EncoderUtil.encodeVLI(blockBB, filterProps.length); + blockBB.put(filterProps); + } + + //make sure there is enough capacity for padding and crc32 + if (blockBB.remaining() < 7) { + blockBB = resize(blockBB); } // Header Padding - while ((bufStream.size() & 3) != 0) - bufStream.write(0x00); + while ((blockBB.position() & 3) != 0) + blockBB.put((byte) 0x00); - byte[] buf = bufStream.toByteArray(); + final int blockSize = blockBB.position(); // Total size of the Block Header: Take the size of the CRC32 field // into account. - headerSize = buf.length + 4; + headerSize = blockBB.position() + 4; // This is just a sanity check. if (headerSize > EncoderUtil.BLOCK_HEADER_SIZE_MAX) throw new UnsupportedOptionsException(); - // Block Header Size - buf[0] = (byte)(buf.length / 4); + // Block Header Size (without crc32 + blockBB.put(0, (byte)(blockSize >> 2)); - // Write the Block Header field to the output stream. - out.write(buf); - EncoderUtil.writeCRC32(out, buf); + //this sets position to 0 and limit to the current position + blockBB.flip(); + + //append the crc32 + EncoderUtil.appendCRC32(blockBB); + assert headerSize == blockBB.position(); + //now write entire block header to the output stream + out.write(blockBB.array(), 0, headerSize); // Calculate the maximum allowed size of the Compressed Data field. // It is hard to exceed it so this is mostly to be pedantic. @@ -85,6 +99,14 @@ class BlockOutputStream extends FinishableOutputStream { - headerSize - check.getSize(); } + private static ByteBuffer resize(ByteBuffer orig) { + int newSize = orig.capacity() << 1; + ByteBuffer newBB = ByteBuffer.allocate(newSize); + orig.flip(); + newBB.put(orig); + return newBB; + } + public void write(int b) throws IOException { tempBuf[0] = (byte)b; write(tempBuf, 0, 1); diff --git a/src/org/tukaani/xz/index/IndexEncoder.java b/src/org/tukaani/xz/index/IndexEncoder.java index 3028802..620b3a4 100644 --- a/src/org/tukaani/xz/index/IndexEncoder.java +++ b/src/org/tukaani/xz/index/IndexEncoder.java @@ -9,13 +9,13 @@ package org.tukaani.xz.index; -import java.io.OutputStream; import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Iterator; -import java.util.zip.CheckedOutputStream; -import org.tukaani.xz.common.EncoderUtil; + import org.tukaani.xz.XZIOException; +import org.tukaani.xz.common.EncoderUtil; public class IndexEncoder extends IndexBase { private final ArrayList<IndexRecord> records @@ -32,28 +32,31 @@ public class IndexEncoder extends IndexBase { } public void encode(OutputStream out) throws IOException { - java.util.zip.CRC32 crc32 = new java.util.zip.CRC32(); - CheckedOutputStream outChecked = new CheckedOutputStream(out, crc32); + + //20 bytes for each record + //16 more bytes for indicator, count of records, padding, and crc32 + final ByteBuffer bb = ByteBuffer.allocate((20 * records.size()) + 16); // Index Indicator - outChecked.write(0x00); + bb.put((byte) 0x00); // Number of Records - EncoderUtil.encodeVLI(outChecked, recordCount); + EncoderUtil.encodeVLI(bb, recordCount); // List of Records for (IndexRecord record : records) { - EncoderUtil.encodeVLI(outChecked, record.unpadded); - EncoderUtil.encodeVLI(outChecked, record.uncompressed); + EncoderUtil.encodeVLI(bb, record.unpadded); + EncoderUtil.encodeVLI(bb, record.uncompressed); } // Index Padding for (int i = getIndexPaddingSize(); i > 0; --i) - outChecked.write(0x00); + bb.put((byte) 0x00); - // CRC32 - long value = crc32.getValue(); - for (int i = 0; i < 4; ++i) - out.write((byte)(value >>> (i * 8))); + // calculate and append CRC32 + assert bb.remaining() >= 4; + bb.flip(); + EncoderUtil.appendCRC32(bb); + out.write(bb.array(), 0, bb.limit()); } } diff --git a/src/org/tukaani/xz/XZOutputStream.java b/src/org/tukaani/xz/XZOutputStream.java index 107ef7f..c927540 100644 --- a/src/org/tukaani/xz/XZOutputStream.java +++ b/src/org/tukaani/xz/XZOutputStream.java @@ -9,11 +9,12 @@ package org.tukaani.xz; -import java.io.OutputStream; import java.io.IOException; +import java.io.OutputStream; + +import org.tukaani.xz.check.Check; import org.tukaani.xz.common.EncoderUtil; import org.tukaani.xz.common.StreamFlags; -import org.tukaani.xz.check.Check; import org.tukaani.xz.index.IndexEncoder; /** @@ -584,22 +585,23 @@ public class XZOutputStream extends FinishableOutputStream { private void encodeStreamHeader() throws IOException { out.write(XZ.HEADER_MAGIC); - byte[] buf = new byte[2]; + byte[] buf = new byte[6]; encodeStreamFlags(buf, 0); - out.write(buf); - EncoderUtil.writeCRC32(out, buf); + EncoderUtil.calcAndInsertCRC32(buf, 0, 2, 2); + out.write(buf); } private void encodeStreamFooter() throws IOException { - byte[] buf = new byte[6]; + //leave 4 bytes at beginning for the crc32 + byte[] buf = new byte[10]; long backwardSize = index.getIndexSize() / 4 - 1; for (int i = 0; i < 4; ++i) - buf[i] = (byte)(backwardSize >>> (i * 8)); + buf[i + 4] = (byte)(backwardSize >>> (i * 8)); - encodeStreamFlags(buf, 4); + encodeStreamFlags(buf, 8); - EncoderUtil.writeCRC32(out, buf); + EncoderUtil.calcAndInsertCRC32(buf, 4, 6, 0); out.write(buf); out.write(XZ.FOOTER_MAGIC); }
