This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 5df260451d TIKA-4889: the inference dispatcher references the
top-level document… (#3176)
5df260451d is described below
commit 5df260451d55a5ecc33f684c4c8e15c7d0bb05af
Author: Tim Allison <[email protected]>
AuthorDate: Tue Sep 15 17:53:04 2026 -0400
TIKA-4889: the inference dispatcher references the top-level document…
(#3176)
---
CHANGES.txt | 7 ++++
.../tika/parser/inference/InferenceDispatcher.java | 23 ++++++++++---
.../parser/inference/InferenceDispatcherTest.java | 38 ++++++++++++++++++++--
3 files changed, 60 insertions(+), 8 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index c2a3962d0f..207eeee4f7 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,12 @@
Release 4.1.0 - unreleased
+ * The inference dispatcher no longer copies the top-level document's own
+ bytes into a buffer file of its own: the stream that owns them is open
+ until after the flush, so the file is referenced. Embedded documents'
+ spools and page renders are still copied, since their parsers delete
+ them first. Groundwork for MEDIA inputs, where the copy was a second
+ full read of an audio or video file (TIKA-4889).
+
* unpack-config gains includeMetadata (TIKA-4881).
* tika-server: /rmeta gains config/{handlerType} (TIKA-4881).
diff --git
a/tika-core/src/main/java/org/apache/tika/parser/inference/InferenceDispatcher.java
b/tika-core/src/main/java/org/apache/tika/parser/inference/InferenceDispatcher.java
index 5b566de340..fd3ace34e2 100644
---
a/tika-core/src/main/java/org/apache/tika/parser/inference/InferenceDispatcher.java
+++
b/tika-core/src/main/java/org/apache/tika/parser/inference/InferenceDispatcher.java
@@ -148,8 +148,11 @@ public final class InferenceDispatcher implements
ParseHook, TransientParseState
}
/**
- * Buffers the bytes at {@code source} for every binding that takes them,
copied into a
- * file the dispatcher owns; dropped past a binding's budget.
+ * Buffers the bytes at {@code source} for every binding that takes them;
dropped past a
+ * binding's budget. The top-level document's own file is referenced,
since the stream
+ * that owns it is open until after the flush; anything else (an embedded
document's
+ * spool, a page render) is copied into a file the dispatcher owns before
its parser
+ * deletes it.
*/
public void offer(InputKind kind, MediaType type, Metadata target,
Metadata parent,
Path source, ParseContext context) throws IOException,
TikaException {
@@ -178,16 +181,26 @@ public final class InferenceDispatcher implements
ParseHook, TransientParseState
continue;
}
if (unit == null) {
- Path copy = state.tmp.createTempFile();
- Files.copy(source, copy, StandardCopyOption.REPLACE_EXISTING);
ParseHooks.Run run = context.get(ParseHooks.Run.class);
unit = new InferenceUnit(kind, type, target, parent,
idPath(target, run),
- parent == null ? null : idPath(parent, run), copy,
page);
+ parent == null ? null : idPath(parent, run),
+ held(source, parent, page, state), page);
}
units.add(unit);
}
}
+ /** The source itself for the top-level document's bytes; a
dispatcher-owned copy otherwise. */
+ private static Path held(Path source, Metadata parent, int page, State
state)
+ throws IOException {
+ if (parent == null && page < 0) {
+ return source;
+ }
+ Path copy = state.tmp.createTempFile();
+ Files.copy(source, copy, StandardCopyOption.REPLACE_EXISTING);
+ return copy;
+ }
+
/** Runs every binding's tasks over its buffered units; clears the buffer
and its files. */
public void flush(Metadata root, ParseContext context) {
State state = context.get(State.class);
diff --git
a/tika-core/src/test/java/org/apache/tika/parser/inference/InferenceDispatcherTest.java
b/tika-core/src/test/java/org/apache/tika/parser/inference/InferenceDispatcherTest.java
index c59c79ae16..265a887f10 100644
---
a/tika-core/src/test/java/org/apache/tika/parser/inference/InferenceDispatcherTest.java
+++
b/tika-core/src/test/java/org/apache/tika/parser/inference/InferenceDispatcherTest.java
@@ -20,6 +20,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -163,9 +164,12 @@ public class InferenceDispatcherTest {
return p;
}
- private void offer(InferenceDispatcher dispatcher, MediaType type, String
bytes,
+ /** Offers as an embedded document: a parent, so the dispatcher copies the
bytes. */
+ private Path offer(InferenceDispatcher dispatcher, MediaType type, String
bytes,
ParseContext context) throws Exception {
- dispatcher.offer(InputKind.IMAGES, type, new Metadata(), null,
file(bytes), context);
+ Path source = file(bytes);
+ dispatcher.offer(InputKind.IMAGES, type, new Metadata(), new
Metadata(), source, context);
+ return source;
}
@Test
@@ -184,13 +188,14 @@ public class InferenceDispatcherTest {
assertTrue(!dispatcher.wants(InputKind.PAGES, PNG, context));
assertTrue(!dispatcher.wants(InputKind.IMAGES, MediaType.image("gif"),
context));
- offer(dispatcher, PNG, "a", context);
+ Path source = offer(dispatcher, PNG, "a", context);
offer(dispatcher, PNG, "b", context);
offer(dispatcher, MediaType.image("jpeg"), "c", context);
offer(dispatcher, MediaType.image("gif"), "d", context);
assertTrue(pngTask.runs.isEmpty(), "nothing runs before the flush");
Path held =
context.get(InferenceDispatcher.State.class).byBinding.get("png").get(0).getPath();
assertTrue(Files.exists(held), "the dispatcher owns a copy until the
flush");
+ assertNotEquals(source, held, "an embedded document's bytes are
copied");
Metadata root = new Metadata();
dispatcher.flush(root, context);
@@ -202,11 +207,38 @@ public class InferenceDispatcherTest {
.contains("skipped 1 units"));
assertNull(context.get(InferenceDispatcher.State.class), "the buffer
is cleared");
assertFalse(Files.exists(held), "and its files are gone");
+ assertTrue(Files.exists(source), "the source was never the
dispatcher's to delete");
dispatcher.flush(root, context);
assertEquals(1, pngTask.runs.size(), "a second flush has nothing to
run");
}
+ /** The top-level document's own file outlives the flush, so it is
referenced, not copied. */
+ @Test
+ public void testTopLevelBytesAreReferencedNotCopied() throws Exception {
+ RecordingTask task = new RecordingTask();
+ InferenceDispatcher dispatcher = new InferenceDispatcher(List.of(
+ new InferenceDispatcher.Bound(binding("png", InputKind.IMAGES,
null, -1),
+ new RecordingEngine(), List.of(task)),
+ new InferenceDispatcher.Bound(binding("pages",
InputKind.PAGES, null, -1),
+ new RecordingEngine(), List.of(task))));
+ ParseContext context = new ParseContext();
+ Path source = file("top");
+ dispatcher.offer(InputKind.IMAGES, PNG, new Metadata(), null, source,
context);
+ InferenceUnit unit =
context.get(InferenceDispatcher.State.class).byBinding.get("png").get(0);
+ assertEquals(source, unit.getPath());
+
+ Path render = file("page");
+ dispatcher.offerPage(PNG, new Metadata(), null, 1, render, context);
+ InferenceUnit page =
context.get(InferenceDispatcher.State.class).byBinding.get("pages").get(0);
+ assertNotEquals(render, page.getPath(), "a page render is the
renderer's file: copied");
+
+ dispatcher.flush(new Metadata(), context);
+ assertArrayEquals("top".getBytes(UTF_8), task.bytes.get(0).get(0));
+ assertTrue(Files.exists(source), "referenced, so not deleted with the
buffer");
+ assertFalse(Files.exists(page.getPath()));
+ }
+
@Test
public void testSelectionAndMaxBytes() throws Exception {
RecordingTask pngTask = new RecordingTask();
