Repository: commons-compress Updated Branches: refs/heads/master 2239893ab -> 68bc083e6
COMPRESS-271 support for skipping skippable frames in lz4 Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/68bc083e Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/68bc083e Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/68bc083e Branch: refs/heads/master Commit: 68bc083e6ba002f7c997f9781f0e24a80d8a3274 Parents: 2239893 Author: Stefan Bodewig <[email protected]> Authored: Sun Feb 5 15:20:37 2017 +0100 Committer: Stefan Bodewig <[email protected]> Committed: Sun Feb 5 15:20:37 2017 +0100 ---------------------------------------------------------------------- .../lz4/FramedLZ4CompressorInputStream.java | 56 ++++- .../lz4/FramedLZ4CompressorInputStreamTest.java | 214 +++++++++++++++++++ 2 files changed, 266 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/commons-compress/blob/68bc083e/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java index 115a2d6..2929bae 100644 --- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java @@ -40,7 +40,6 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream { /* * TODO before releasing 1.14: * - * + skippable frames * + block dependence */ @@ -48,6 +47,10 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream { static final byte[] LZ4_SIGNATURE = new byte[] { //NOSONAR 4, 0x22, 0x4d, 0x18 }; + private static final byte[] SKIPPABLE_FRAME_TRAILER = new byte[] { + 0x2a, 0x4d, 0x18 + }; + private static final byte SKIPPABLE_FRAME_PREFIX_BYTE_MASK = 0x50; static final int VERSION_MASK = 0xC0; static final int SUPPORTED_VERSION = 0x40; @@ -151,15 +154,25 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream { } private boolean readSignature(boolean firstFrame) throws IOException { + String garbageMessage = firstFrame ? "Not a LZ4 frame stream" : "LZ4 frame stream followed by garbage"; final byte[] b = new byte[4]; - final int read = IOUtils.readFully(in, b); + int read = IOUtils.readFully(in, b); count(read); - if (4 != read && !firstFrame) { + if (0 == read && !firstFrame) { + endReached = true; + return false; + } + if (4 != read) { + throw new IOException(garbageMessage); + } + + read = skipSkippableFrame(b); + if (0 == read && !firstFrame) { endReached = true; return false; } if (4 != read || !matches(b, 4)) { - throw new IOException("Not a LZ4 frame stream"); + throw new IOException(garbageMessage); } return true; } @@ -281,6 +294,41 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream { } } + private static boolean isSkippableFrameSignature(byte[] b) { + if ((b[0] & SKIPPABLE_FRAME_PREFIX_BYTE_MASK) != SKIPPABLE_FRAME_PREFIX_BYTE_MASK) { + return false; + } + for (int i = 1; i < 4; i++) { + if (b[i] != SKIPPABLE_FRAME_TRAILER[i - 1]) { + return false; + } + } + return true; + } + + /** + * Skips over the contents of a skippable frame as well as + * skippable frames following it. + * + * <p>It then tries to read four more bytes which are supposed to + * hold an LZ4 signature and returns the number of bytes read + * while storing the bytes in the given array.</p> + */ + private int skipSkippableFrame(byte[] b) throws IOException { + int read = 4; + while (read == 4 && isSkippableFrameSignature(b)) { + long len = ByteUtils.fromLittleEndian(supplier, 4); + long skipped = IOUtils.skip(in, len); + count(skipped); + if (len != skipped) { + throw new IOException("Premature end of stream while skipping frame"); + } + read = IOUtils.readFully(in, b); + count(read); + } + return read; + } + /** * Checks if the signature matches what is expected for a .lz4 file. * http://git-wip-us.apache.org/repos/asf/commons-compress/blob/68bc083e/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java index 0363e5e..9dda432 100644 --- a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java +++ b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java @@ -362,6 +362,220 @@ public final class FramedLZ4CompressorInputStreamTest } } + @Test + public void skipsOverSkippableFrames() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + 0x5f, 0x2a, 0x4d, 0x18, // skippable frame signature + 2, 0, 0, 0, // skippable frame has length 2 + 1, 2, // content of skippable frame + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 1, 0, 0, (byte) 0x80, // 1 bytes length and uncompressed bit set + '!', // content + 0, 0, 0, 0, // empty block marker + }; + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) { + byte[] actual = IOUtils.toByteArray(a); + assertArrayEquals(new byte[] { + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', '!' + }, actual); + } + } + + @Test + public void skipsOverTrailingSkippableFrames() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + 0x51, 0x2a, 0x4d, 0x18, // skippable frame signature + 2, 0, 0, 0, // skippable frame has length 2 + 1, 2, // content of skippable frame + }; + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) { + byte[] actual = IOUtils.toByteArray(a); + assertArrayEquals(new byte[] { + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!' + }, actual); + } + } + + @Test + public void rejectsSkippableFrameFollowedByJunk() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + 0x50, 0x2a, 0x4d, 0x18, // skippable frame signature + 2, 0, 0, 0, // skippable frame has length 2 + 1, 2, // content of skippable frame + 1, 0x22, 0x4d, 0x18, // bad signature + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) { + IOUtils.toByteArray(a); + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("garbage")); + } + } + + @Test + public void rejectsSkippableFrameFollowedByTooFewBytes() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + 0x52, 0x2a, 0x4d, 0x18, // skippable frame signature + 2, 0, 0, 0, // skippable frame has length 2 + 1, 2, // content of skippable frame + 4, // too short for signature + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) { + IOUtils.toByteArray(a); + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("garbage")); + } + } + + @Test + public void rejectsSkippableFrameWithPrematureEnd() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + 0x50, 0x2a, 0x4d, 0x18, // skippable frame signature + 2, 0, 0, 0, // skippable frame has length 2 + 1, // content of skippable frame (should be two bytes) + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) { + IOUtils.toByteArray(a); + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("Premature end of stream while skipping frame")); + } + } + + @Test + public void rejectsSkippableFrameWithPrematureEndInLengthBytes() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + 0x55, 0x2a, 0x4d, 0x18, // skippable frame signature + 2, 0, 0, // should be four byte length + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) { + IOUtils.toByteArray(a); + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("premature end of data")); + } + } + + @Test + public void rejectsSkippableFrameWithBadSignatureTrailer() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + 0x51, 0x2a, 0x4d, 0x17, // broken skippable frame signature + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) { + IOUtils.toByteArray(a); + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("garbage")); + } + } + + @Test + public void rejectsSkippableFrameWithBadSignaturePrefix() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + 0x60, 0x2a, 0x4d, 0x18, // broken skippable frame signature + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) { + IOUtils.toByteArray(a); + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("garbage")); + } + } + + @Test + public void rejectsTrailingBytesAfterValidFrame() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum + 0x70, // block size 4MB + 115, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + 0x56, 0x2a, 0x4d, // too short for any signature + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input), true)) { + IOUtils.toByteArray(a); + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("garbage")); + } + } + interface StreamWrapper { InputStream wrap(InputStream in) throws Exception; }
