This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 726bb2a841 TIKA-4878: pdf and pst (#3133)
726bb2a841 is described below
commit 726bb2a841a6bdfef942afef15cc8424f355e91d
Author: Tim Allison <[email protected]>
AuthorDate: Fri Sep 4 14:04:24 2026 -0400
TIKA-4878: pdf and pst (#3133)
---
CHANGES.txt | 7 ++
.../parser/microsoft/pst/PSTMailItemParser.java | 18 ++-
.../apache/tika/parser/pdf/AbstractPDF2XHTML.java | 19 ++-
.../parser/pdf/PDFEmbeddedFileNoTempFileTest.java | 135 +++++++++++++++++++++
4 files changed, 167 insertions(+), 12 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index c24adef94e..a3aaaebafd 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,12 @@
Release 4.1.0 - unreleased
+ * PDF attachments, PDF XMP packets, 3D on-instantiate scripts and PST
+ attachments are re-opened from their document instead of cached when
+ the embedded-document extractor rewinds them (digesting does, for every
+ embedded document). The cached copy cost heap for the whole attachment
+ and, past the cache budget or the 1 MB floor, a temp file; PDFBox and
+ java-libpst hand the bytes back on demand (TIKA-4878).
+
* tika-server: named configuration presets (TIKA-4856).
* Temp files follow -Djava.io.tmpdir on the parent JVM (Tika, its
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/pst/PSTMailItemParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/pst/PSTMailItemParser.java
index 509000babb..72b85d73af 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/pst/PSTMailItemParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/pst/PSTMailItemParser.java
@@ -35,6 +35,7 @@ import org.apache.tika.annotation.TikaComponent;
import org.apache.tika.exception.TikaException;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.extractor.EmbeddedDocumentUtil;
+import org.apache.tika.io.TemporaryResources;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.MAPI;
@@ -264,18 +265,23 @@ public class PSTMailItemParser implements Parser {
attributes.addAttribute("", "id", "id", "CDATA", filename);
xhtml.startElement("div", attributes);
if (embeddedExtractor.shouldParseEmbedded(attachMeta, context)) {
- TikaInputStream tis = null;
+ //open once now so a broken attachment is recorded here, not
mid-parse
try {
- tis = TikaInputStream.get(attachment.getFileInputStream());
+ attachment.getFileInputStream().close();
} catch (NullPointerException e) { //TIKA-2488
EmbeddedDocumentUtil.recordEmbeddedStreamException(e,
metadata, context);
return;
}
-
- try {
+ //the attachment is in the pst already: re-open it on rewind
rather than
+ //cache a copy that a digest of a large attachment would spill to
disk
+ try (TikaInputStream tis = TikaInputStream.get(() -> {
+ try {
+ return attachment.getFileInputStream();
+ } catch (PSTException e) {
+ throw new IOException(e);
+ }
+ }, new TemporaryResources(), null)) {
embeddedExtractor.parseEmbedded(tis, xhtml, attachMeta,
context, false);
- } finally {
- tis.close();
}
}
xhtml.endElement("div");
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
index 7ad8d52e56..c3b60f8ab4 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
@@ -282,7 +282,8 @@ class AbstractPDF2XHTML extends PDFTextStripper {
//try the main metadata
if (pdDocument.getDocumentCatalog().getMetadata() != null) {
try (TikaInputStream tis = TikaInputStream.get(
-
pdDocument.getDocumentCatalog().getMetadata().exportXMPMetadata())) {
+ () ->
pdDocument.getDocumentCatalog().getMetadata().exportXMPMetadata(),
+ new TemporaryResources(), null)) {
extractXMPAsEmbeddedFile(tis,
XMP_DOCUMENT_CATALOG_LOCATION);
} catch (IOException e) {
EmbeddedDocumentUtil.recordEmbeddedStreamException(e,
metadata, context);
@@ -292,7 +293,9 @@ class AbstractPDF2XHTML extends PDFTextStripper {
int pageNumber = 1;
for (PDPage page : pdDocument.getPages()) {
if (page.getMetadata() != null) {
- try (TikaInputStream tis =
TikaInputStream.get(page.getMetadata().exportXMPMetadata())) {
+ try (TikaInputStream tis = TikaInputStream.get(
+ () -> page.getMetadata().exportXMPMetadata(),
+ new TemporaryResources(), null)) {
extractXMPAsEmbeddedFile(tis, XMP_PAGE_LOCATION_PREFIX
+ pageNumber);
} catch (IOException e) {
EmbeddedDocumentUtil.recordEmbeddedStreamException(e,
metadata, context);
@@ -500,14 +503,17 @@ class AbstractPDF2XHTML extends PDFTextStripper {
if (!embeddedDocumentExtractor.shouldParseEmbedded(embeddedMetadata,
context)) {
return;
}
- TikaInputStream tis = null;
+ //open once now so a broken stream is recorded here, not mid-parse
try {
- tis = TikaInputStream.get(pdEmbeddedFile.createInputStream());
+ pdEmbeddedFile.createInputStream().close();
} catch (IOException e) {
- //store this exception in the parent's metadata
EmbeddedDocumentUtil.recordEmbeddedStreamException(e, metadata,
context);
return;
}
+ //the file is in the document already: re-open (re-decode) it on rewind
+ //rather than cache a copy that a digest of a large attachment would
spill
+ TikaInputStream tis =
TikaInputStream.get(pdEmbeddedFile::createInputStream,
+ new TemporaryResources(), null);
setOrReplaceAttribute("class", "embedded", attributes);
setOrReplaceAttribute("id", fileName, attributes);
@@ -945,7 +951,8 @@ class AbstractPDF2XHTML extends PDFTextStripper {
}
Metadata m = getJavascriptMetadata("3DD_ON_INSTANTIATE", null, null);
if (embeddedDocumentExtractor.shouldParseEmbedded(m, context)) {
- try (TikaInputStream tis =
TikaInputStream.get(stream.createInputStream())) {
+ try (TikaInputStream tis =
TikaInputStream.get(stream::createInputStream,
+ new TemporaryResources(), null)) {
embeddedDocumentExtractor.parseEmbedded(tis, xhtml, m,
context, true);
}
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFEmbeddedFileNoTempFileTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFEmbeddedFileNoTempFileTest.java
new file mode 100644
index 0000000000..03ddc6b343
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFEmbeddedFileNoTempFileTest.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.pdf;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.pdfbox.cos.COSName;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDDocumentNameDictionary;
+import org.apache.pdfbox.pdmodel.PDEmbeddedFilesNameTreeNode;
+import org.apache.pdfbox.pdmodel.PDPage;
+import
org.apache.pdfbox.pdmodel.common.filespecification.PDComplexFileSpecification;
+import org.apache.pdfbox.pdmodel.common.filespecification.PDEmbeddedFile;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.extractor.EmbeddedDocumentExtractor;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+
+/**
+ * A PDF attachment's bytes are in the document already. Rewinding the stream
+ * handed to the embedded-document extractor -- which digesting does for every
+ * embedded document -- must re-open (re-decode) the attachment from the
+ * document, not cache a copy of it and spill that copy to a temp file.
+ * <p>
+ * The payload is over the 1 MB a cache keeps in memory, so a cached stream has
+ * to spill to rewind and the difference is observable. The assertion is on the
+ * stream the extractor is handed: the parser owns the child's
+ * {@code TemporaryResources}, so a watched directory would pass either way.
+ */
+public class PDFEmbeddedFileNoTempFileTest extends TikaTest {
+
+ private static final int PAYLOAD_LENGTH = 2 * 1024 * 1024;
+
+ @TempDir
+ Path tempDir;
+
+ @Test
+ public void testAttachmentIsNotSpooled() throws Exception {
+ Path pdf = tempDir.resolve("attachment.pdf");
+ byte[] payload = payload();
+ try (PDDocument doc = new PDDocument()) {
+ doc.addPage(new PDPage());
+ //Flate-encoded, as real attachments are: a rewind has to re-decode
+ PDEmbeddedFile file = new PDEmbeddedFile(doc, new
ByteArrayInputStream(payload),
+ COSName.FLATE_DECODE);
+ file.setSize(payload.length);
+ PDComplexFileSpecification spec = new PDComplexFileSpecification();
+ spec.setFile("attachment.bin");
+ spec.setEmbeddedFile(file);
+ PDEmbeddedFilesNameTreeNode tree = new
PDEmbeddedFilesNameTreeNode();
+ tree.setNames(Map.of("attachment.bin", spec));
+ PDDocumentNameDictionary names = new
PDDocumentNameDictionary(doc.getDocumentCatalog());
+ names.setEmbeddedFiles(tree);
+ doc.getDocumentCatalog().setNames(names);
+ doc.save(pdf.toFile());
+ }
+
+ RecordingExtractor extractor = new RecordingExtractor();
+ ParseContext context = new ParseContext();
+ context.set(EmbeddedDocumentExtractor.class, extractor);
+ Metadata metadata = new Metadata();
+ try (TikaInputStream tis = TikaInputStream.get(pdf, metadata)) {
+ new PDFParser().parse(tis, new DefaultHandler(), metadata,
context);
+ }
+
+ assertTrue(extractor.lengths.contains(PAYLOAD_LENGTH),
+ "the attachment reached the extractor in full; saw " +
extractor.lengths);
+ for (int i = 0; i < extractor.spooled.size(); i++) {
+ assertEquals(false, extractor.spooled.get(i), "embedded stream " +
i + " ("
+ + extractor.lengths.get(i)
+ + " bytes) was spooled to disk to rewind instead of
re-opened");
+ }
+ }
+
+ /** Incompressible filler, so Flate keeps it at full size. */
+ private static byte[] payload() {
+ byte[] bytes = new byte[PAYLOAD_LENGTH];
+ new Random(4878).nextBytes(bytes);
+ return bytes;
+ }
+
+ /**
+ * Rewinds each embedded stream the way a digester does, then records
whether
+ * that left it backed by a temp file and how many bytes it still yields.
+ */
+ private static class RecordingExtractor implements
EmbeddedDocumentExtractor {
+ private final List<Boolean> spooled = new ArrayList<>();
+ private final List<Integer> lengths = new ArrayList<>();
+
+ @Override
+ public boolean shouldParseEmbedded(Metadata metadata, ParseContext
context) {
+ return true;
+ }
+
+ @Override
+ public void parseEmbedded(TikaInputStream stream, ContentHandler
handler,
+ Metadata metadata, ParseContext context,
boolean outputHtml)
+ throws IOException {
+ stream.enableRewind();
+ stream.readAllBytes();
+ stream.rewind();
+ spooled.add(stream.hasFile());
+ lengths.add(stream.readAllBytes().length);
+ }
+ }
+}
