This is an automated email from the ASF dual-hosted git repository. mthomsen pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push: new 3e60414d48 NIFI-10436 Initial basic brotli support NIFI-10436 Add Brotli-compressed SampleFile.txt.br for unit testing NIFI-10436 Fixup basic usage of Brotli4j API NIFI-10436 NOTICE ok, LICENSE updated NIFI-10436 style fixup CompressContent.java NIFI-10436 Update CompressContent.java 3e60414d48 is described below commit 3e60414d486342ee59cee65818c79f3612946d3d Author: Matthew Hawkins <hawko2...@gmail.com> AuthorDate: Sat Sep 17 21:13:30 2022 +1000 NIFI-10436 Initial basic brotli support NIFI-10436 Add Brotli-compressed SampleFile.txt.br for unit testing NIFI-10436 Fixup basic usage of Brotli4j API NIFI-10436 NOTICE ok, LICENSE updated NIFI-10436 style fixup CompressContent.java NIFI-10436 Update CompressContent.java This closes #6432 Signed-off-by: Mike Thomsen <mthom...@apache.org> --- nifi-assembly/LICENSE | 5 +++- .../nifi-standard-processors/pom.xml | 5 ++++ .../nifi/processors/standard/CompressContent.java | 29 +++++++++++++++++--- .../processors/standard/TestCompressContent.java | 30 +++++++++++++++++++++ .../resources/CompressedData/SampleFile.txt.br | Bin 0 -> 71 bytes nifi-nar-bundles/nifi-standard-bundle/pom.xml | 5 ++++ 6 files changed, 69 insertions(+), 5 deletions(-) diff --git a/nifi-assembly/LICENSE b/nifi-assembly/LICENSE index 4ecd35df8a..e6842290a4 100644 --- a/nifi-assembly/LICENSE +++ b/nifi-assembly/LICENSE @@ -3120,4 +3120,7 @@ which is available under an MIT license. AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. \ No newline at end of file + THE SOFTWARE. + +This product bundles 'Brotli4j' which is available under the Apache License 2.0 + diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml index 3ea60c1647..0a7b197ae8 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml @@ -229,6 +229,11 @@ <groupId>com.github.luben</groupId> <artifactId>zstd-jni</artifactId> </dependency> + <dependency> + <groupId>com.aayushatharva.brotli4j</groupId> + <artifactId>brotli4j</artifactId> + <version>1.8.0</version> + </dependency> <dependency> <groupId>org.tukaani</groupId> <artifactId>xz</artifactId> diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/CompressContent.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/CompressContent.java index 6ad4192cd1..605b25cf97 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/CompressContent.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/CompressContent.java @@ -25,6 +25,10 @@ import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorInputStream; import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream; import org.apache.commons.compress.compressors.zstandard.ZstdCompressorOutputStream; +import com.aayushatharva.brotli4j.decoder.BrotliInputStream; +import com.aayushatharva.brotli4j.encoder.BrotliOutputStream; +import com.aayushatharva.brotli4j.Brotli4jLoader; +import com.aayushatharva.brotli4j.encoder.Encoder; import org.apache.nifi.annotation.behavior.EventDriven; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; @@ -84,7 +88,7 @@ import java.util.zip.InflaterInputStream; @SideEffectFree @SupportsBatching @InputRequirement(Requirement.INPUT_REQUIRED) -@Tags({"content", "compress", "decompress", "gzip", "bzip2", "lzma", "xz-lzma2", "snappy", "snappy-hadoop", "snappy framed", "lz4-framed", "deflate", "zstd"}) +@Tags({"content", "compress", "decompress", "gzip", "bzip2", "lzma", "xz-lzma2", "snappy", "snappy-hadoop", "snappy framed", "lz4-framed", "deflate", "zstd", "brotli"}) @CapabilityDescription("Compresses or decompresses the contents of FlowFiles using a user-specified compression algorithm and updates the mime.type " + "attribute as appropriate. This processor operates in a very memory efficient way so very large objects well beyond the heap size " + "are generally fine to process") @@ -107,16 +111,17 @@ public class CompressContent extends AbstractProcessor { public static final String COMPRESSION_FORMAT_SNAPPY_FRAMED = "snappy framed"; public static final String COMPRESSION_FORMAT_LZ4_FRAMED ="lz4-framed"; public static final String COMPRESSION_FORMAT_ZSTD = "zstd"; + public static final String COMPRESSION_FORMAT_BROTLI = "brotli"; public static final String MODE_COMPRESS = "compress"; public static final String MODE_DECOMPRESS = "decompress"; public static final PropertyDescriptor COMPRESSION_FORMAT = new PropertyDescriptor.Builder() .name("Compression Format") - .description("The compression format to use. Valid values are: GZIP, Deflate, ZSTD, BZIP2, XZ-LZMA2, LZMA, Snappy, Snappy Hadoop, Snappy Framed, and LZ4-Framed") + .description("The compression format to use. Valid values are: GZIP, Deflate, ZSTD, BZIP2, XZ-LZMA2, LZMA, Brotli, Snappy, Snappy Hadoop, Snappy Framed, and LZ4-Framed") .allowableValues(COMPRESSION_FORMAT_ATTRIBUTE, COMPRESSION_FORMAT_GZIP, COMPRESSION_FORMAT_DEFLATE, COMPRESSION_FORMAT_BZIP2, COMPRESSION_FORMAT_XZ_LZMA2, COMPRESSION_FORMAT_LZMA, COMPRESSION_FORMAT_SNAPPY, COMPRESSION_FORMAT_SNAPPY_HADOOP, COMPRESSION_FORMAT_SNAPPY_FRAMED, - COMPRESSION_FORMAT_LZ4_FRAMED, COMPRESSION_FORMAT_ZSTD) + COMPRESSION_FORMAT_LZ4_FRAMED, COMPRESSION_FORMAT_ZSTD, COMPRESSION_FORMAT_BROTLI) .defaultValue(COMPRESSION_FORMAT_ATTRIBUTE) .required(true) .build(); @@ -135,7 +140,8 @@ public class CompressContent extends AbstractProcessor { .defaultValue("1") .required(true) .allowableValues("0", "1", "2", "3", "4", "5", "6", "7", "8", "9") - .dependsOn(COMPRESSION_FORMAT, COMPRESSION_FORMAT_ATTRIBUTE, COMPRESSION_FORMAT_GZIP, COMPRESSION_FORMAT_DEFLATE, COMPRESSION_FORMAT_XZ_LZMA2, COMPRESSION_FORMAT_ZSTD) + .dependsOn(COMPRESSION_FORMAT, COMPRESSION_FORMAT_ATTRIBUTE, COMPRESSION_FORMAT_GZIP, COMPRESSION_FORMAT_DEFLATE, + COMPRESSION_FORMAT_XZ_LZMA2, COMPRESSION_FORMAT_ZSTD, COMPRESSION_FORMAT_BROTLI) .dependsOn(MODE, MODE_COMPRESS) .build(); @@ -188,6 +194,7 @@ public class CompressContent extends AbstractProcessor { mimeTypeMap.put("application/x-snappy-framed", COMPRESSION_FORMAT_SNAPPY_FRAMED); mimeTypeMap.put("application/x-lz4-framed", COMPRESSION_FORMAT_LZ4_FRAMED); mimeTypeMap.put("application/zstd", COMPRESSION_FORMAT_ZSTD); + mimeTypeMap.put("application/x-brotli", COMPRESSION_FORMAT_BROTLI); this.compressionFormatMimeTypeMap = Collections.unmodifiableMap(mimeTypeMap); } @@ -280,6 +287,9 @@ public class CompressContent extends AbstractProcessor { case COMPRESSION_FORMAT_ZSTD: fileExtension = ".zst"; break; + case COMPRESSION_FORMAT_BROTLI: + fileExtension = ".br"; + break; default: fileExtension = ""; break; @@ -340,6 +350,13 @@ public class CompressContent extends AbstractProcessor { compressionOut = new ZstdCompressorOutputStream(bufferedOut, zstdcompressionLevel); mimeTypeRef.set("application/zstd"); break; + case COMPRESSION_FORMAT_BROTLI: + Brotli4jLoader.ensureAvailability(); + compressionLevel = context.getProperty(COMPRESSION_LEVEL).asInteger(); + Encoder.Parameters params = new Encoder.Parameters().setQuality(compressionLevel); + compressionOut = new BrotliOutputStream(bufferedOut, params); + mimeTypeRef.set("application/x-brotli"); + break; case COMPRESSION_FORMAT_BZIP2: default: mimeTypeRef.set("application/x-bzip2"); @@ -379,6 +396,10 @@ public class CompressContent extends AbstractProcessor { case COMPRESSION_FORMAT_ZSTD: compressionIn = new ZstdCompressorInputStream(bufferedIn); break; + case COMPRESSION_FORMAT_BROTLI: + Brotli4jLoader.ensureAvailability(); + compressionIn = new BrotliInputStream(bufferedIn); + break; default: compressionIn = new CompressorStreamFactory().createCompressorInputStream(compressionFormat.toLowerCase(), bufferedIn); } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestCompressContent.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestCompressContent.java index 0d8727a8e3..a1cb5feadc 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestCompressContent.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestCompressContent.java @@ -377,4 +377,34 @@ public class TestCompressContent { flowFile.assertContentEquals(Paths.get("src/test/resources/CompressedData/SampleFile.txt")); flowFile.assertAttributeEquals("filename", "SampleFile.txt"); } + + @Test + public void testBrotliCompress() throws Exception { + final TestRunner runner = TestRunners.newTestRunner(CompressContent.class); + runner.setProperty(CompressContent.MODE, CompressContent.MODE_COMPRESS); + runner.setProperty(CompressContent.COMPRESSION_FORMAT, CompressContent.COMPRESSION_FORMAT_BROTLI); + runner.setProperty(CompressContent.UPDATE_FILENAME, "true"); + + runner.enqueue(Paths.get("src/test/resources/CompressedData/SampleFile.txt")); + runner.run(); + + runner.assertAllFlowFilesTransferred(CompressContent.REL_SUCCESS, 1); + MockFlowFile flowFile = runner.getFlowFilesForRelationship(CompressContent.REL_SUCCESS).get(0); + flowFile.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "application/x-brotli"); + flowFile.assertAttributeEquals("filename", "SampleFile.txt.br"); + } + + @Test + public void testBrotliDecompress() throws Exception { + final TestRunner runner = TestRunners.newTestRunner(CompressContent.class); + runner.setProperty(CompressContent.MODE, CompressContent.MODE_DECOMPRESS); + runner.setProperty(CompressContent.COMPRESSION_FORMAT, CompressContent.COMPRESSION_FORMAT_BROTLI); + runner.setProperty(CompressContent.UPDATE_FILENAME, "true"); + runner.enqueue(Paths.get("src/test/resources/CompressedData/SampleFile.txt.br")); + runner.run(); + runner.assertAllFlowFilesTransferred(CompressContent.REL_SUCCESS, 1); + MockFlowFile flowFile = runner.getFlowFilesForRelationship(CompressContent.REL_SUCCESS).get(0); + flowFile.assertContentEquals(Paths.get("src/test/resources/CompressedData/SampleFile.txt")); + flowFile.assertAttributeEquals("filename", "SampleFile.txt"); + } } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/CompressedData/SampleFile.txt.br b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/CompressedData/SampleFile.txt.br new file mode 100755 index 0000000000..dd2bbc8f8d Binary files /dev/null and b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/CompressedData/SampleFile.txt.br differ diff --git a/nifi-nar-bundles/nifi-standard-bundle/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/pom.xml index 1890838f68..166b013ff3 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/pom.xml @@ -159,6 +159,11 @@ <artifactId>zstd-jni</artifactId> <version>1.5.2-3</version> </dependency> + <dependency> + <groupId>com.aayushatharva.brotli4j</groupId> + <artifactId>brotli4j</artifactId> + <version>1.8.0</version> + </dependency> <dependency> <groupId>org.tukaani</groupId> <artifactId>xz</artifactId>