Should it be "Z_STANDARD" instead of "ZSTANDARD" since "standard" is a word?
Gary On Tue, Oct 17, 2017 at 12:23 PM, <bode...@apache.org> wrote: > Repository: commons-compress > Updated Branches: > refs/heads/master 89bc17055 -> 1c382914c > > > COMPRESS-423 - Add ZStandard decompression support using Zstd-JNI > > > Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo > Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/ > commit/7984387a > Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/ > tree/7984387a > Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/ > diff/7984387a > > Branch: refs/heads/master > Commit: 7984387af004fcfe1d1ee12e2c8e6b68f23be001 > Parents: 89bc170 > Author: Andre F de Miranda <trix...@users.noreply.github.com> > Authored: Sat Oct 14 17:57:19 2017 +1100 > Committer: Stefan Bodewig <bode...@apache.org> > Committed: Tue Oct 17 20:17:01 2017 +0200 > > ---------------------------------------------------------------------- > pom.xml | 6 + > .../compressors/CompressorStreamFactory.java | 25 ++- > .../zstandard/ZstdCompressorInputStream.java | 95 +++++++++++ > .../compressors/zstandard/ZstdUtils.java | 88 +++++++++++ > .../ZstdCompressorInputStreamTest.java | 157 +++++++++++++++++++ > src/test/resources/bla.tar.zst | Bin 0 -> 473 bytes > src/test/resources/zstandard.testdata | 3 + > src/test/resources/zstandard.testdata.zst | Bin 0 -> 94 bytes > 8 files changed, 372 insertions(+), 2 deletions(-) > ---------------------------------------------------------------------- > > > http://git-wip-us.apache.org/repos/asf/commons-compress/ > blob/7984387a/pom.xml > ---------------------------------------------------------------------- > diff --git a/pom.xml b/pom.xml > index 31fc4cd..6a33b38 100644 > --- a/pom.xml > +++ b/pom.xml > @@ -74,6 +74,12 @@ jar, tar, zip, dump, 7z, arj. > <scope>test</scope> > </dependency> > <dependency> > + <groupId>com.github.luben</groupId> > + <artifactId>zstd-jni</artifactId> > + <version>1.3.1-1</version> > + <optional>true</optional> > + </dependency> > + <dependency> > <groupId>org.brotli</groupId> > <artifactId>dec</artifactId> > <version>0.1.2</version> > > http://git-wip-us.apache.org/repos/asf/commons-compress/ > blob/7984387a/src/main/java/org/apache/commons/compress/compressors/ > CompressorStreamFactory.java > ---------------------------------------------------------------------- > diff --git > a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java > b/src/main/java/org/apache/commons/compress/compressors/ > CompressorStreamFactory.java > index 4bd22aa..b446963 100644 > --- a/src/main/java/org/apache/commons/compress/compressors/ > CompressorStreamFactory.java > +++ b/src/main/java/org/apache/commons/compress/compressors/ > CompressorStreamFactory.java > @@ -55,6 +55,8 @@ import org.apache.commons.compress.compressors.xz. > XZCompressorInputStream; > import org.apache.commons.compress.compressors.xz. > XZCompressorOutputStream; > import org.apache.commons.compress.compressors.xz.XZUtils; > import org.apache.commons.compress.compressors.z.ZCompressorInputStream; > +import org.apache.commons.compress.compressors.zstandard. > ZstdCompressorInputStream; > +import org.apache.commons.compress.compressors.zstandard.ZstdUtils; > import org.apache.commons.compress.utils.IOUtils; > import org.apache.commons.compress.utils.Lists; > import org.apache.commons.compress.utils.ServiceLoaderIterator; > @@ -191,6 +193,14 @@ public class CompressorStreamFactory implements > CompressorStreamProvider { > public static final String LZ4_FRAMED = "lz4-framed"; > > /** > + * Constant (value {@value}) used to identify the ZStandard > compression > + * algorithm. Not supported as an output stream type. > + * > + * @since 1.15 > + */ > + public static final String ZSTANDARD = "zst"; > + > + /** > * Constructs a new sorted map from input stream provider names to > provider > * objects. > * > @@ -279,7 +289,7 @@ public class CompressorStreamFactory implements > CompressorStreamProvider { > public static String getBrotli() { > return BROTLI; > } > - > + > public static String getBzip2() { > return BZIP2; > } > @@ -328,6 +338,10 @@ public class CompressorStreamFactory implements > CompressorStreamProvider { > return LZ4_BLOCK; > } > > + public static String getZstandard() { > + return ZSTANDARD; > + } > + > static void putAll(final Set<String> names, final > CompressorStreamProvider provider, > final TreeMap<String, CompressorStreamProvider> map) { > for (final String name : names) { > @@ -555,6 +569,13 @@ public class CompressorStreamFactory implements > CompressorStreamProvider { > return new XZCompressorInputStream(in, > actualDecompressConcatenated, memoryLimitInKb); > } > > + if (ZSTANDARD.equalsIgnoreCase(name)) { > + if (!ZstdUtils.isZstdCompressionAvailable()) { > + throw new CompressorException("XZ compression is not > available."); > + } > + return new ZstdCompressorInputStream(in); > + } > + > if (LZMA.equalsIgnoreCase(name)) { > if (!LZMAUtils.isLZMACompressionAvailable()) { > throw new CompressorException("LZMA compression is > not available"); > @@ -701,7 +722,7 @@ public class CompressorStreamFactory implements > CompressorStreamProvider { > @Override > public Set<String> getInputStreamCompressorNames() { > return Sets.newHashSet(GZIP, BROTLI, BZIP2, XZ, LZMA, PACK200, > DEFLATE, SNAPPY_RAW, SNAPPY_FRAMED, Z, LZ4_BLOCK, > - LZ4_FRAMED); > + LZ4_FRAMED, ZSTANDARD); > } > > @Override > > http://git-wip-us.apache.org/repos/asf/commons-compress/ > blob/7984387a/src/main/java/org/apache/commons/compress/ > compressors/zstandard/ZstdCompressorInputStream.java > ---------------------------------------------------------------------- > diff --git a/src/main/java/org/apache/commons/compress/compressors/ > zstandard/ZstdCompressorInputStream.java b/src/main/java/org/apache/ > commons/compress/compressors/zstandard/ZstdCompressorInputStream.java > new file mode 100644 > index 0000000..1e5dd8d > --- /dev/null > +++ b/src/main/java/org/apache/commons/compress/compressors/zstandard/ > ZstdCompressorInputStream.java > @@ -0,0 +1,95 @@ > +/* > + * Licensed to the Apache Software Foundation (ASF) under one or more > + * contributor license agreements. See the NOTICE file distributed with > + * this work for additional information regarding copyright ownership. > + * The ASF licenses this file to You under the Apache License, Version 2.0 > + * (the "License"); you may not use this file except in compliance with > + * the License. You may obtain a copy of the License at > + * > + * http://www.apache.org/licenses/LICENSE-2.0 > + * > + * Unless required by applicable law or agreed to in writing, software > + * distributed under the License is distributed on an "AS IS" BASIS, > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or > implied. > + * See the License for the specific language governing permissions and > + * limitations under the License. > + */ > + > +package org.apache.commons.compress.compressors.zstandard; > + > + > +import java.io.IOException; > +import java.io.InputStream; > + > +import com.github.luben.zstd.ZstdInputStream; > +import org.apache.commons.compress.compressors.CompressorInputStream; > + > +/** > + * {@link CompressorInputStream} implementation to decode Zstandard > encoded stream. > + * Library relies on <a href="https://github.com/luben/zstd-jni/";>Zstandard > JNI</a> > + * > + * @since 1.15 > + */ > +public class ZstdCompressorInputStream extends CompressorInputStream { > + > + private final com.github.luben.zstd.ZstdInputStream decIS; > + > + public ZstdCompressorInputStream(final InputStream in) throws > IOException { > + this.decIS = new ZstdInputStream(in); > + } > + > + @Override > + public int available() throws IOException { > + return decIS.available(); > + } > + > + @Override > + public void close() throws IOException { > + decIS.close(); > + } > + > + @Override > + public int read(final byte[] b) throws IOException { > + return decIS.read(b); > + } > + > + @Override > + public long skip(final long n) throws IOException { > + return decIS.skip(n); > + } > + > + @Override > + public void mark(final int readlimit) { > + decIS.mark(readlimit); > + } > + > + @Override > + public boolean markSupported() { > + return decIS.markSupported(); > + } > + > + @Override > + public int read() throws IOException { > + final int ret = decIS.read(); > + count(ret == -1 ? 0 : 1); > + return ret; > + } > + > + @Override > + public int read(final byte[] buf, final int off, final int len) > throws IOException { > + final int ret = decIS.read(buf, off, len); > + count(ret); > + return ret; > + } > + > + @Override > + public String toString() { > + return decIS.toString(); > + } > + > + @Override > + public void reset() throws IOException { > + decIS.reset(); > + } > + > +} > \ No newline at end of file > > http://git-wip-us.apache.org/repos/asf/commons-compress/ > blob/7984387a/src/main/java/org/apache/commons/compress/ > compressors/zstandard/ZstdUtils.java > ---------------------------------------------------------------------- > diff --git > a/src/main/java/org/apache/commons/compress/compressors/zstandard/ZstdUtils.java > b/src/main/java/org/apache/commons/compress/compressors/ > zstandard/ZstdUtils.java > new file mode 100644 > index 0000000..0eb8fa1 > --- /dev/null > +++ b/src/main/java/org/apache/commons/compress/compressors/ > zstandard/ZstdUtils.java > @@ -0,0 +1,88 @@ > +/* > + * Licensed to the Apache Software Foundation (ASF) under one > + * or more contributor license agreements. See the NOTICE file > + * distributed with this work for additional information > + * regarding copyright ownership. The ASF licenses this file > + * to you under the Apache License, Version 2.0 (the > + * "License"); you may not use this file except in compliance > + * with the License. You may obtain a copy of the License at > + * > + * http://www.apache.org/licenses/LICENSE-2.0 > + * > + * Unless required by applicable law or agreed to in writing, > + * software distributed under the License is distributed on an > + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY > + * KIND, either express or implied. See the License for the > + * specific language governing permissions and limitations > + * under the License. > + */ > +package org.apache.commons.compress.compressors.zstandard; > + > +/** > + * Utility code for the Zstandard compression format. > + * @ThreadSafe > + * @since 1.14 > + */ > +public class ZstdUtils { > + > + static enum CachedAvailability { > + DONT_CACHE, CACHED_AVAILABLE, CACHED_UNAVAILABLE > + } > + > + private static volatile CachedAvailability cachedZstdAvailability; > + > + static { > + cachedZstdAvailability = CachedAvailability.DONT_CACHE; > + try { > + Class.forName("org.osgi.framework.BundleEvent"); > + } catch (final Exception ex) { // NOSONAR > + setCacheZstdAvailablity(true); > + } > + } > + > + /** Private constructor to prevent instantiation of this utility > class. */ > + private ZstdUtils() { > + } > + > + /** > + * Are the classes required to support Zstandard compression > available? > + * @return true if the classes required to support Zstandard > compression are available > + */ > + public static boolean isZstdCompressionAvailable() { > + final CachedAvailability cachedResult = cachedZstdAvailability; > + if (cachedResult != CachedAvailability.DONT_CACHE) { > + return cachedResult == CachedAvailability.CACHED_AVAILABLE; > + } > + return internalIsZstdCompressionAvailable(); > + } > + > + private static boolean internalIsZstdCompressionAvailable() { > + try { > + Class.forName("com.github.luben.zstd.ZstdInputStream"); > + return true; > + } catch (NoClassDefFoundError | Exception error) { > + return false; > + } > + } > + > + /** > + * Whether to cache the result of the Zstandard for Java check. > + * > + * <p>This defaults to {@code false} in an OSGi environment and > {@code true} otherwise.</p> > + * @param doCache whether to cache the result > + */ > + public static void setCacheZstdAvailablity(final boolean doCache) { > + if (!doCache) { > + cachedZstdAvailability = CachedAvailability.DONT_CACHE; > + } else if (cachedZstdAvailability == > CachedAvailability.DONT_CACHE) { > + final boolean hasZstd = internalIsZstdCompressionAvailable(); > + cachedZstdAvailability = hasZstd ? CachedAvailability.CACHED_ > AVAILABLE > + : CachedAvailability.CACHED_UNAVAILABLE; > + } > + } > + > + // only exists to support unit tests > + static CachedAvailability getCachedZstdAvailability() { > + return cachedZstdAvailability; > + } > +} > > http://git-wip-us.apache.org/repos/asf/commons-compress/ > blob/7984387a/src/test/java/org/apache/commons/compress/ > compressors/zstandard/ZstdCompressorInputStreamTest.java > ---------------------------------------------------------------------- > diff --git a/src/test/java/org/apache/commons/compress/compressors/ > zstandard/ZstdCompressorInputStreamTest.java b/src/test/java/org/apache/ > commons/compress/compressors/zstandard/ZstdCompressorInputStreamTest.java > new file mode 100644 > index 0000000..5ed276c > --- /dev/null > +++ b/src/test/java/org/apache/commons/compress/compressors/zstandard/ > ZstdCompressorInputStreamTest.java > @@ -0,0 +1,157 @@ > +/* > + * Licensed to the Apache Software Foundation (ASF) under one or more > + * contributor license agreements. See the NOTICE file distributed with > + * this work for additional information regarding copyright ownership. > + * The ASF licenses this file to You under the Apache License, Version 2.0 > + * (the "License"); you may not use this file except in compliance with > + * the License. You may obtain a copy of the License at > + * > + * http://www.apache.org/licenses/LICENSE-2.0 > + * > + * Unless required by applicable law or agreed to in writing, software > + * distributed under the License is distributed on an "AS IS" BASIS, > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or > implied. > + * See the License for the specific language governing permissions and > + * limitations under the License. > + */ > + > +package org.apache.commons.compress.compressors.zstandard; > + > +import static org.junit.Assert.assertEquals; > +import static org.junit.Assert.assertTrue; > + > +import java.io.ByteArrayOutputStream; > +import java.io.File; > +import java.io.FileInputStream; > +import java.io.FileOutputStream; > +import java.io.IOException; > +import java.io.InputStream; > + > +import org.apache.commons.compress.AbstractTestCase; > +import org.apache.commons.compress.compressors.CompressorInputStream; > +import org.apache.commons.compress.compressors.CompressorStreamFactory; > +import org.apache.commons.compress.compressors.zstandard. > ZstdCompressorInputStream; > +import org.apache.commons.compress.utils.IOUtils; > +import org.junit.Assert; > +import org.junit.Test; > + > +public class ZstdCompressorInputStreamTest extends AbstractTestCase { > + > + /** > + * Test bridge works fine > + * @throws {@link IOException} > + */ > + @Test > + public void testZstdDecode() throws IOException { > + final File input = getFile("zstandard.testdata.zst"); > + final File expected = getFile("zstandard.testdata"); > + try (InputStream inputStream = new FileInputStream(input); > + InputStream expectedStream = new FileInputStream(expected); > + ZstdCompressorInputStream zstdInputStream = new > ZstdCompressorInputStream(inputStream)) { > + final byte[] b = new byte[97]; > + IOUtils.readFully(expectedStream, b); > + final ByteArrayOutputStream bos = new ByteArrayOutputStream(); > + int readByte = -1; > + while((readByte = zstdInputStream.read()) != -1) { > + bos.write(readByte); > + } > + Assert.assertArrayEquals(b, bos.toByteArray()); > + } > + } > + > + @Test > + public void testCachingIsEnabledByDefaultAndZstdUtilsPresent() { > + assertEquals(ZstdUtils.CachedAvailability.CACHED_AVAILABLE, > ZstdUtils.getCachedZstdAvailability()); > + assertTrue(ZstdUtils.isZstdCompressionAvailable()); > + } > + > + @Test > + public void testCanTurnOffCaching() { > + try { > + ZstdUtils.setCacheZstdAvailablity(false); > + assertEquals(ZstdUtils.CachedAvailability.DONT_CACHE, > ZstdUtils.getCachedZstdAvailability()); > + assertTrue(ZstdUtils.isZstdCompressionAvailable()); > + } finally { > + ZstdUtils.setCacheZstdAvailablity(true); > + } > + } > + > + @Test > + public void testTurningOnCachingReEvaluatesAvailability() { > + try { > + ZstdUtils.setCacheZstdAvailablity(false); > + assertEquals(ZstdUtils.CachedAvailability.DONT_CACHE, > ZstdUtils.getCachedZstdAvailability()); > + ZstdUtils.setCacheZstdAvailablity(true); > + assertEquals(ZstdUtils.CachedAvailability.CACHED_AVAILABLE, > ZstdUtils.getCachedZstdAvailability()); > + } finally { > + ZstdUtils.setCacheZstdAvailablity(true); > + } > + } > + > + @Test > + public void shouldBeAbleToSkipAByte() throws IOException { > + final File input = getFile("zstandard.testdata.zst"); > + try (InputStream is = new FileInputStream(input)) { > + final ZstdCompressorInputStream in = > + new ZstdCompressorInputStream(is); > + Assert.assertEquals(1, in.skip(1)); > + in.close(); > + } > + } > + > + @Test > + public void singleByteReadWorksAsExpected() throws IOException { > + > + final File input = getFile("zstandard.testdata.zst"); > + > + final File original = getFile("zstandard.testdata"); > + final long originalFileLength = original.length(); > + > + byte[] originalFileContent = new byte[((int) originalFileLength)]; > + > + try (InputStream ois = new FileInputStream(original)) { > + ois.read(originalFileContent); > + } > + > + try (InputStream is = new FileInputStream(input)) { > + final ZstdCompressorInputStream in = > + new ZstdCompressorInputStream(is); > + > + Assert.assertEquals(originalFileContent[0], in.read()); > + in.close(); > + } > + } > + > + @Test > + public void singleByteReadReturnsMinusOneAtEof() throws IOException { > + final File input = getFile("zstandard.testdata.zst"); > + try (InputStream is = new FileInputStream(input)) { > + final ZstdCompressorInputStream in = > + new ZstdCompressorInputStream(is); > + IOUtils.toByteArray(in); > + Assert.assertEquals(-1, in.read()); > + in.close(); > + } > + } > + > + @Test > + public void testZstandardUnarchive() throws Exception { > + final File input = getFile("bla.tar.zst"); > + final File output = new File(dir, "bla.tar"); > + try (InputStream is = new FileInputStream(input)) { > + final CompressorInputStream in = new CompressorStreamFactory() > + .createCompressorInputStream("zst", is); > + FileOutputStream out = null; > + try { > + out = new FileOutputStream(output); > + IOUtils.copy(in, out); > + } finally { > + if (out != null) { > + out.close(); > + } > + in.close(); > + } > + } > + } > + > +} > > http://git-wip-us.apache.org/repos/asf/commons-compress/ > blob/7984387a/src/test/resources/bla.tar.zst > ---------------------------------------------------------------------- > diff --git a/src/test/resources/bla.tar.zst b/src/test/resources/bla.tar. > zst > new file mode 100644 > index 0000000..d5fd6e0 > Binary files /dev/null and b/src/test/resources/bla.tar.zst differ > > http://git-wip-us.apache.org/repos/asf/commons-compress/ > blob/7984387a/src/test/resources/zstandard.testdata > ---------------------------------------------------------------------- > diff --git a/src/test/resources/zstandard.testdata b/src/test/resources/ > zstandard.testdata > new file mode 100644 > index 0000000..e51bfd4 > --- /dev/null > +++ b/src/test/resources/zstandard.testdata > @@ -0,0 +1,3 @@ > +And as usual, instead of ipsum lorem we shall just state very clearly: > + > +Test test test chocolate > > http://git-wip-us.apache.org/repos/asf/commons-compress/ > blob/7984387a/src/test/resources/zstandard.testdata.zst > ---------------------------------------------------------------------- > diff --git a/src/test/resources/zstandard.testdata.zst > b/src/test/resources/zstandard.testdata.zst > new file mode 100644 > index 0000000..86c03fe > Binary files /dev/null and b/src/test/resources/zstandard.testdata.zst > differ > >
