This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch branch_3x
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/branch_3x by this push:
new 174b518099 TIKA-4989: file-system emitter can gzip its output;
tika-eval finds .json.gz extracts (backport) (#3394)
174b518099 is described below
commit 174b51809948b912a3cb6715bac6690d41b50fbb
Author: Tim Allison <[email protected]>
AuthorDate: Thu Oct 22 09:18:39 2026 -0400
TIKA-4989: file-system emitter can gzip its output; tika-eval finds
.json.gz extracts (backport) (#3394)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
CHANGES.txt | 4 +
.../org/apache/tika/eval/app/AbstractProfiler.java | 7 +-
.../org/apache/tika/eval/app/EvalFilePaths.java | 4 +-
.../org/apache/tika/eval/app/io/ExtractReader.java | 32 +++++-
.../tika/eval/app/ComparerGzipExtractsTest.java | 117 +++++++++++++++++++++
.../tika/pipes/emitter/fs/FileSystemEmitter.java | 66 +++++++++++-
.../pipes/emitter/fs/FileSystemEmitterTest.java | 88 ++++++++++++++++
7 files changed, 312 insertions(+), 6 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 4aeee01703..13d44cd96c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,9 @@
Release 3.4.0 - ???
+ * The file-system emitter can gzip its output (param "compression": "gzip"),
and tika-eval
+ finds gzipped extracts (.json.gz) when comparing against a plain extract
set, measuring
+ their extract lengths and min/max length limits on uncompressed size
(TIKA-4989).
+
* tika-server caps attachments on the multipart/form-data endpoints at 50 MB
by
default; set <maxAttachmentBytes> in the server config to raise it or -1 to
remove it (CVE-2026-54225). The form-parameter cap from CVE-2026-57819
diff --git
a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/AbstractProfiler.java
b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/AbstractProfiler.java
index 3455fc5375..a2704a56c9 100644
---
a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/AbstractProfiler.java
+++
b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/AbstractProfiler.java
@@ -47,6 +47,7 @@ import org.apache.tika.batch.fs.FSProperties;
import org.apache.tika.eval.app.db.ColInfo;
import org.apache.tika.eval.app.db.Cols;
import org.apache.tika.eval.app.db.TableInfo;
+import org.apache.tika.eval.app.io.ExtractReader;
import org.apache.tika.eval.app.io.ExtractReaderException;
import org.apache.tika.eval.app.io.IDBWriter;
import org.apache.tika.eval.core.langid.LanguageIDWrapper;
@@ -88,7 +89,7 @@ public abstract class AbstractProfiler extends
FileResourceConsumer {
private static final Property CONTAINER_EXCEPTION_1X =
Property.externalText("X-TIKA" + ":EXCEPTION:runtime");
private static final Logger LOG =
LoggerFactory.getLogger(AbstractProfiler.class);
private static final String[] EXTRACT_EXTENSIONS = {".json", ".txt", ""};
- private static final String[] COMPRESSION_EXTENSIONS = {"", ".bz2",
".gzip", ".zip",};
+ private static final String[] COMPRESSION_EXTENSIONS = {"", ".gz", ".bz2",
".gzip", ".zip",};
private static final String ZERO = "0";
private static final String UNKNOWN_EXTENSION = "unk";
//make this configurable
@@ -105,7 +106,7 @@ public abstract class AbstractProfiler extends
FileResourceConsumer {
public static TableInfo MIME_TABLE = new TableInfo("mimes", new
ColInfo(Cols.MIME_ID, Types.INTEGER, "PRIMARY KEY"), new
ColInfo(Cols.MIME_STRING, Types.VARCHAR, 256),
new ColInfo(Cols.FILE_EXTENSION, Types.VARCHAR, 12));
private static CommonTokenCountManager COMMON_TOKEN_COUNT_MANAGER;
- private static Pattern FILE_NAME_CLEANER =
Pattern.compile("\\.(json|txt)(\\.(bz2|gz|zip))?$");
+ private static Pattern FILE_NAME_CLEANER =
Pattern.compile("\\.(json|txt)(\\.(bz2|gz|gzip|zip))?$");
private static LanguageIDWrapper LANG_ID = new LanguageIDWrapper();
protected IDBWriter writer;
AnalyzerManager analyzerManager;
@@ -795,7 +796,7 @@ public abstract class AbstractProfiler extends
FileResourceConsumer {
protected long getFileLength(Path p) {
if (p != null && Files.isRegularFile(p)) {
try {
- return Files.size(p);
+ return ExtractReader.extractLength(p);
} catch (IOException e) {
//swallow
}
diff --git
a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/EvalFilePaths.java
b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/EvalFilePaths.java
index 769a72b48c..bf3fe351f8 100644
---
a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/EvalFilePaths.java
+++
b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/EvalFilePaths.java
@@ -23,6 +23,8 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Objects;
+import org.apache.tika.eval.app.io.ExtractReader;
+
/**
* Simple struct to keep track of relative path of source file (
* original binary file, e.g. /subdir/document1.doc)
@@ -45,7 +47,7 @@ class EvalFilePaths {
public EvalFilePaths(Path relativeSourceFilePath, Path extractFile) {
if (extractFile != null && Files.isRegularFile(extractFile)) {
try {
- extractFileLength = Files.size(extractFile);
+ extractFileLength = ExtractReader.extractLength(extractFile);
} catch (IOException e) {
//swallow ?
}
diff --git
a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/ExtractReader.java
b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/ExtractReader.java
index bfab8b253f..d807fcf8ac 100644
---
a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/ExtractReader.java
+++
b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/ExtractReader.java
@@ -21,9 +21,13 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
@@ -90,6 +94,32 @@ public class ExtractReader {
return fileSuffixes;
}
+ /**
+ * Uncompressed length for gzip (from the ISIZE trailer), so plain and
gzipped extract sets
+ * agree on lengths and min/max filtering; on-disk length for everything
else.
+ */
+ public static long extractLength(Path extractFile) throws IOException {
+ long onDisk = Files.size(extractFile);
+ String name =
extractFile.getFileName().toString().toLowerCase(Locale.ROOT);
+ if (onDisk < 18 || !(name.endsWith(".gz") || name.endsWith(".gzip"))) {
+ return onDisk;
+ }
+ ByteBuffer trailer =
ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN);
+ try (FileChannel ch = FileChannel.open(extractFile,
StandardOpenOption.READ)) {
+ while (trailer.hasRemaining()) {
+ if (ch.read(trailer, onDisk - 4 + trailer.position()) < 0) {
+ return onDisk;
+ }
+ }
+ }
+ long isize = trailer.getInt(0) & 0xFFFFFFFFL;
+ // smaller than even stored blocks could yield: wrapped past 4 GiB or
multi-member
+ if (isize + (onDisk >> 10) + 64 < onDisk) {
+ return onDisk;
+ }
+ return isize;
+ }
+
public List<Metadata> loadExtract(Path extractFile) throws
ExtractReaderException {
List<Metadata> metadataList = null;
@@ -109,7 +139,7 @@ public class ExtractReader {
long length = -1L;
try {
- length = Files.size(extractFile);
+ length = extractLength(extractFile);
} catch (IOException e) {
throw new
ExtractReaderException(ExtractReaderException.TYPE.IO_EXCEPTION, e);
}
diff --git
a/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/ComparerGzipExtractsTest.java
b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/ComparerGzipExtractsTest.java
new file mode 100644
index 0000000000..9e0918d6f7
--- /dev/null
+++
b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/ComparerGzipExtractsTest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.app;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import org.apache.tika.eval.app.db.H2Util;
+
+/** A gzipped extract set ({@code .json.gz}, as the file-system emitter writes
it) compares like its plain twin. */
+public class ComparerGzipExtractsTest {
+
+ private static final String[] QUERIES = {
+ "select file_path || ' ' || extract_file_length_a || ' ' ||
extract_file_length_b"
+ + " from containers order by file_path",
+ "select count(1) from profiles_a",
+ "select count(1) from profiles_b",
+ "select count(1) from extract_exceptions_a",
+ "select count(1) from extract_exceptions_b",
+ "select count(1) from content_comparisons"
+ };
+
+ @TempDir
+ Path tmp;
+
+ @Test
+ public void testGzipOnEitherSide() throws Exception {
+ Path testDirs =
Paths.get(getClass().getResource("/test-dirs").toURI());
+ Path a = testDirs.resolve("extractsA");
+ Path b = testDirs.resolve("extractsB");
+ List<String> plain = compare("plain", a, b);
+ assertTrue(plain.size() > QUERIES.length, "comparison produced no
containers");
+ assertEquals(plain, compare("gzB", a, gzipCopy(b,
tmp.resolve("bGz"))), "gzipped B");
+ assertEquals(plain, compare("gzA", gzipCopy(a, tmp.resolve("aGz")),
b), "gzipped A");
+ }
+
+ private List<String> compare(String name, Path a, Path b) throws Exception
{
+ Path db = tmp.resolve(name).toAbsolutePath();
+ List<String> args = new ArrayList<>();
+ args.add("java");
+ args.add("-Djava.awt.headless=true");
+ args.add("-cp");
+ args.add(System.getProperty("java.class.path"));
+ args.add(TikaEvalCLI.class.getName());
+ args.add("Compare");
+ args.add("-extractsA");
+ args.add(a.toAbsolutePath().toString());
+ args.add("-extractsB");
+ args.add(b.toAbsolutePath().toString());
+ args.add("-db");
+ args.add(db.toString());
+ Process process = new ProcessBuilder(args).inheritIO().start();
+ assertTrue(process.waitFor(2, TimeUnit.MINUTES), "compare timed out");
+ assertEquals(0, process.exitValue());
+
+ List<String> results = new ArrayList<>();
+ try (Connection c = new H2Util(db).getConnection(); Statement st =
c.createStatement()) {
+ for (String q : QUERIES) {
+ try (ResultSet rs = st.executeQuery(q)) {
+ while (rs.next()) {
+ results.add(q + " -> " + rs.getString(1));
+ }
+ }
+ }
+ }
+ return results;
+ }
+
+ private static Path gzipCopy(Path src, Path dest) throws IOException {
+ FileUtils.copyDirectory(src.toFile(), dest.toFile());
+ List<Path> jsons;
+ try (Stream<Path> files = Files.walk(dest)) {
+ jsons = files.filter(f ->
f.toString().endsWith(".json")).collect(Collectors.toList());
+ }
+ for (Path p : jsons) {
+ try (OutputStream os = new GZIPOutputStream(
+ Files.newOutputStream(p.resolveSibling(p.getFileName() +
".gz")))) {
+ Files.copy(p, os);
+ }
+ Files.delete(p);
+ }
+ return dest;
+ }
+}
diff --git
a/tika-pipes/tika-emitters/tika-emitter-fs/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitter.java
b/tika-pipes/tika-emitters/tika-emitter-fs/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitter.java
index 0da31981cf..332e52cd1b 100644
---
a/tika-pipes/tika-emitters/tika-emitter-fs/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitter.java
+++
b/tika-pipes/tika-emitters/tika-emitter-fs/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitter.java
@@ -16,8 +16,11 @@
*/
package org.apache.tika.pipes.emitter.fs;
+import java.io.BufferedWriter;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileAlreadyExistsException;
@@ -26,6 +29,8 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.List;
+import java.util.Locale;
+import java.util.zip.GZIPOutputStream;
import org.apache.tika.config.Field;
import org.apache.tika.metadata.Metadata;
@@ -60,6 +65,9 @@ import org.apache.tika.serialization.JsonMetadataList;
* <!-- optional; whether or not to pretty print the output
* default is false -->
* <param name="prettyPrint"
type="boolean">true</param>
+ * <!-- optional; 'none' or 'gzip'; gzip appends '.gz' to
the file name
+ * default is 'none' -->
+ * <param name="compression"
type="string">gzip</param>
* </params>
* </emitter>
* </emitters>
@@ -67,12 +75,16 @@ import org.apache.tika.serialization.JsonMetadataList;
*/
public class FileSystemEmitter extends AbstractEmitter implements
StreamEmitter {
+ private static final int GZIP_BUFFER_SIZE = 64 * 1024;
+
private Path basePath = null;
private String fileExtension = "json";
private ON_EXISTS onExists = ON_EXISTS.EXCEPTION;
private boolean prettyPrint = false;
+ private COMPRESSION compression = COMPRESSION.NONE;
+
@Override
public void emit(String emitKey, List<Metadata> metadataList, ParseContext
parseContext) throws IOException, TikaEmitterException {
Path output;
@@ -83,6 +95,9 @@ public class FileSystemEmitter extends AbstractEmitter
implements StreamEmitter
if (fileExtension != null && ! fileExtension.isEmpty()) {
emitKey += "." + fileExtension;
}
+ if (compression == COMPRESSION.GZIP) {
+ emitKey += ".gz";
+ }
if (basePath != null) {
output = basePath.resolve(emitKey);
if
(!output.toAbsolutePath().normalize().startsWith(basePath.toAbsolutePath().normalize()))
{
@@ -95,11 +110,36 @@ public class FileSystemEmitter extends AbstractEmitter
implements StreamEmitter
if (output.getParent() != null &&
!Files.isDirectory(output.getParent())) {
Files.createDirectories(output.getParent());
}
- try (Writer writer = Files.newBufferedWriter(output,
StandardCharsets.UTF_8)) {
+ try (Writer writer = newWriter(output)) {
JsonMetadataList.toJson(metadataList, writer, prettyPrint);
}
}
+ private Writer newWriter(Path output) throws IOException {
+ if (compression != COMPRESSION.GZIP) {
+ return Files.newBufferedWriter(output, StandardCharsets.UTF_8);
+ }
+ return new BufferedWriter(new
OutputStreamWriter(gzip(Files.newOutputStream(output)),
+ StandardCharsets.UTF_8));
+ }
+
+ static OutputStream gzip(OutputStream os) throws IOException {
+ try {
+ return new GZIPOutputStream(os, GZIP_BUFFER_SIZE) {
+ // GZIPOutputStream.close() skips closing os when finish()
throws (e.g. disk full)
+ @Override
+ public void close() throws IOException {
+ try (os) {
+ super.close();
+ }
+ }
+ };
+ } catch (IOException e) {
+ os.close();
+ throw e;
+ }
+ }
+
@Field
public void setBasePath(String basePath) {
this.basePath = Paths.get(basePath);
@@ -145,6 +185,26 @@ public class FileSystemEmitter extends AbstractEmitter
implements StreamEmitter
this.prettyPrint = prettyPrint;
}
+ /**
+ * Compression for the json metadata list output: 'none' (default) or
'gzip'.
+ * Does not apply to {@link #emit(String, InputStream, Metadata,
ParseContext)}.
+ *
+ * @param compression
+ */
+ @Field
+ public void setCompression(String compression) {
+ switch (compression.toLowerCase(Locale.ROOT)) {
+ case "none":
+ this.compression = COMPRESSION.NONE;
+ break;
+ case "gzip":
+ this.compression = COMPRESSION.GZIP;
+ break;
+ default:
+ throw new IllegalArgumentException("Don't understand '" +
compression + "'; must be one of: 'none', 'gzip'");
+ }
+ }
+
@Override
public void emit(String path, InputStream inputStream, Metadata
userMetadata, ParseContext parseContext) throws IOException,
TikaEmitterException {
Path target = basePath.resolve(path);
@@ -170,4 +230,8 @@ public class FileSystemEmitter extends AbstractEmitter
implements StreamEmitter
enum ON_EXISTS {
SKIP, EXCEPTION, REPLACE
}
+
+ enum COMPRESSION {
+ NONE, GZIP
+ }
}
diff --git
a/tika-pipes/tika-emitters/tika-emitter-fs/src/test/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitterTest.java
b/tika-pipes/tika-emitters/tika-emitter-fs/src/test/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitterTest.java
new file mode 100644
index 0000000000..bde4818603
--- /dev/null
+++
b/tika-pipes/tika-emitters/tika-emitter-fs/src/test/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitterTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.emitter.fs;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+import java.util.zip.GZIPInputStream;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.serialization.JsonMetadataList;
+
+public class FileSystemEmitterTest {
+
+ @TempDir
+ Path tmp;
+
+ @Test
+ public void testGzip() throws Exception {
+ FileSystemEmitter emitter = emitter("GZIP");
+ emitter.emit("sub/a.pdf", metadataList(), new ParseContext());
+
+ Path gz = tmp.resolve("sub/a.pdf.json.gz");
+ assertTrue(Files.isRegularFile(gz));
+ assertFalse(Files.exists(tmp.resolve("sub/a.pdf.json")));
+ try (Reader r = new InputStreamReader(new
GZIPInputStream(Files.newInputStream(gz)),
+ StandardCharsets.UTF_8)) {
+ List<Metadata> back = JsonMetadataList.fromJson(r);
+ assertEquals("hello world",
back.get(0).get(TikaCoreProperties.TIKA_CONTENT));
+ }
+ }
+
+ @Test
+ public void testDefaultIsUncompressed() throws Exception {
+ FileSystemEmitter emitter = emitter("none");
+ emitter.emit("a.pdf", metadataList(), new ParseContext());
+ try (Reader r = Files.newBufferedReader(tmp.resolve("a.pdf.json"),
StandardCharsets.UTF_8)) {
+ assertEquals("hello world",
+
JsonMetadataList.fromJson(r).get(0).get(TikaCoreProperties.TIKA_CONTENT));
+ }
+ }
+
+ @Test
+ public void testBadCompression() {
+ assertThrows(IllegalArgumentException.class, () -> new
FileSystemEmitter().setCompression("zstd"));
+ }
+
+ private FileSystemEmitter emitter(String compression) {
+ FileSystemEmitter emitter = new FileSystemEmitter();
+ emitter.setBasePath(tmp.toString());
+ emitter.setCompression(compression);
+ return emitter;
+ }
+
+ private static List<Metadata> metadataList() {
+ Metadata m = new Metadata();
+ m.set(TikaCoreProperties.TIKA_CONTENT, "hello world");
+ return Collections.singletonList(m);
+ }
+}
