This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new ef4d529533 TIKA-4887: a PDF page render is enriched once, by the PDF
parser (#3151)
ef4d529533 is described below
commit ef4d5295337c50ad8173ced418260b45342f7c58
Author: Tim Allison <[email protected]>
AuthorDate: Thu Sep 10 19:20:38 2026 -0400
TIKA-4887: a PDF page render is enriched once, by the PDF parser (#3151)
---
CHANGES.txt | 10 ++
docs/modules/ROOT/pages/configuration/index.adoc | 2 +
.../pages/configuration/parsers/pdf-parser.adoc | 7 +
.../tika/parser/enricher/ContentEnrichers.java | 93 ++++++++++++-
.../tika/parser/enricher/ContentEnrichersTest.java | 60 +++++++++
.../apache/tika/parser/pdf/AbstractPDF2XHTML.java | 69 ++++++++--
.../java/org/apache/tika/parser/pdf/OCR2XHTML.java | 4 +-
.../java/org/apache/tika/parser/pdf/PDF2XHTML.java | 5 +-
.../java/org/apache/tika/parser/pdf/PDFParser.java | 22 +--
.../org/apache/tika/parser/pdf/PDFParserTest.java | 149 +++++++++++++++++++++
10 files changed, 392 insertions(+), 29 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index b8381770c7..8c6512827a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,15 @@
Release 4.1.0 - unreleased
+ * A PDF page rendered as an embedded document (imageStrategy RENDER_PAGES_*)
+ is enriched once, by the PDF parser: the text recognizer where the OCR
+ strategy says so, every other text-recognizers entry on every page,
results on
+ the PDF itself. The embedded copy is no longer enriched again by
+ image-parser, so an image embedder no longer yields two vectors for a
+ text-poor page and Tesseract no longer OCRs a page twice; NO_OCR now means
+ no page OCR at all. ContentEnrichers gains scoped suspend and
+ suspendRecognizers for containers that enrich their own renderings
+ (TIKA-4887).
+
* OCR engines and other content enrichers: name the engine in
"text-recognizers"
and every default parser stays loaded; "text-recognizers": [] turns
enrichment
off everywhere. With no list, the loader resolves one engine
diff --git a/docs/modules/ROOT/pages/configuration/index.adoc
b/docs/modules/ROOT/pages/configuration/index.adoc
index 2b5d8b914f..34f4219083 100644
--- a/docs/modules/ROOT/pages/configuration/index.adoc
+++ b/docs/modules/ROOT/pages/configuration/index.adoc
@@ -206,6 +206,8 @@ declares it through its `textRecognizer` option (default
`true`). An entry that
text (the experimental image-embedding parser, a VLM with `textRecognizer:
false`) still runs on
the images and pages it is offered and never displaces text; startup warns
about it, and 4.2
gives such annotators a list of their own.
+When the PDF parser also emits its page renders as embedded documents, it
enriches each render itself and
+the embedded copy is not enriched again (see
xref:configuration/parsers/pdf-parser.adoc[PDF Parser]).
=== For engine authors
diff --git a/docs/modules/ROOT/pages/configuration/parsers/pdf-parser.adoc
b/docs/modules/ROOT/pages/configuration/parsers/pdf-parser.adoc
index d4215e96a7..16d386b406 100644
--- a/docs/modules/ROOT/pages/configuration/parsers/pdf-parser.adoc
+++ b/docs/modules/ROOT/pages/configuration/parsers/pdf-parser.adoc
@@ -65,6 +65,13 @@ Inline images embedded in a PDF are a separate path: with
`extractInlineImages`
parsed as embedded documents, so `image-parser` enriches each one with the
same engine. A scanned
page that is one full-page image is therefore OCR'd twice when both are on.
+Page renders emitted as embedded documents (`imageStrategy` set to
`RENDER_PAGES_AT_PAGE_END` or
+`RENDER_PAGES_BEFORE_PARSE`) are enriched by the PDF parser itself, once per
page: the text
+recognizer where `strategy` says so, and every other enricher (an image
embedder, a tagging VLM)
+on every page, with its results such as `tk:chunks` landing on the PDF. The
embedded copy of a
+render is parsed for its metadata but not enriched again, so no page reaches
the same engine
+twice, and `NO_OCR` means no page OCR anywhere while the other enrichers still
see every page.
+
`pdf:ocr-page-count` records how many pages tripped the verdict, whether or
not an engine ran.
Every `ocr` option can be overridden per request through `parse-context` under
`pdf-parser`; the
xref:using-tika/server/index.adoc[server documentation] shows the `NO_OCR`
override as a preset.
diff --git
a/tika-core/src/main/java/org/apache/tika/parser/enricher/ContentEnrichers.java
b/tika-core/src/main/java/org/apache/tika/parser/enricher/ContentEnrichers.java
index da57fb2d35..b1a8329312 100644
---
a/tika-core/src/main/java/org/apache/tika/parser/enricher/ContentEnrichers.java
+++
b/tika-core/src/main/java/org/apache/tika/parser/enricher/ContentEnrichers.java
@@ -92,11 +92,45 @@ public final class ContentEnrichers {
public static Parser get(CompositeContentEnricher enrichers, MediaType
mediaType,
Metadata target, ParseContext context) {
Objects.requireNonNull(target, "target");
- if (mediaType == null || isActive(context)) {
+ return select(enrichers, mediaType, context);
+ }
+
+ /**
+ * Suspends dispatch in this context until the returned scope closes. A
container that
+ * enriches a rendering itself wraps the embedded parse of that rendering,
so the embedded
+ * copy is not enriched a second time. Scopes nest; each restores the
state it found.
+ */
+ public static Suspension suspend(ParseContext context) {
+ return new Suspension(context, false);
+ }
+
+ /**
+ * Suspends text recognizers only, until the returned scope closes: {@link
#get} returns
+ * the annotators (embedders, taggers, captioners) and {@link
#hasTextRecognizer} is
+ * false. A recognizer that declines this parse is still a recognizer, not
an annotator.
+ * Scopes nest as for {@link #suspend}.
+ */
+ public static Suspension suspendRecognizers(ParseContext context) {
+ return new Suspension(context, true);
+ }
+
+ private static Parser select(CompositeContentEnricher enrichers, MediaType
mediaType,
+ ParseContext context) {
+ if (mediaType == null || isActive(context) || isSuspended(context)) {
return null;
}
+ boolean annotatorsOnly = isRecognizersSuspended(context);
if (enrichers != null) {
List<Parser> matched = enrichers.getEnrichers(mediaType);
+ if (annotatorsOnly) {
+ List<Parser> annotators = new ArrayList<>();
+ for (Parser p : matched) {
+ if (isAnnotator(p, enrichers.isLegacyClaimant(p))) {
+ annotators.add(p);
+ }
+ }
+ matched = annotators;
+ }
if (matched.isEmpty()) {
return null;
}
@@ -104,7 +138,15 @@ public final class ContentEnrichers {
? matched.get(0) : new SequentialEnricher(matched));
}
Candidate discovered = discover(mediaType, context);
- return discovered == null ? null : new
GuardedEnricher(discovered.parser);
+ if (discovered == null
+ || (annotatorsOnly && !isAnnotator(discovered.parser,
discovered.legacy))) {
+ return null;
+ }
+ return new GuardedEnricher(discovered.parser);
+ }
+
+ private static boolean isAnnotator(Parser member, boolean legacy) {
+ return !legacy && asTextRecognizer(member) == null;
}
/**
@@ -117,7 +159,8 @@ public final class ContentEnrichers {
MediaType mediaType, Metadata
target,
ParseContext context) {
Objects.requireNonNull(target, "target");
- if (mediaType == null || isActive(context)) {
+ if (mediaType == null || isActive(context) || isSuspended(context)
+ || isRecognizersSuspended(context)) {
return false;
}
if (enrichers != null) {
@@ -224,6 +267,46 @@ public final class ContentEnrichers {
return active != null && active.active;
}
+ private static boolean isSuspended(ParseContext context) {
+ ActiveEnrichment state = context.get(ActiveEnrichment.class);
+ return state != null && state.suspended;
+ }
+
+ private static boolean isRecognizersSuspended(ParseContext context) {
+ ActiveEnrichment state = context.get(ActiveEnrichment.class);
+ return state != null && state.recognizersSuspended;
+ }
+
+ /** Scope of a {@link #suspend} or {@link #suspendRecognizers}; closing
restores the prior state. */
+ public static final class Suspension implements AutoCloseable {
+
+ private final ActiveEnrichment state;
+ private final boolean wasSuspended;
+ private final boolean wereRecognizersSuspended;
+
+ private Suspension(ParseContext context, boolean recognizersOnly) {
+ ActiveEnrichment found = context.get(ActiveEnrichment.class);
+ if (found == null) {
+ found = new ActiveEnrichment();
+ context.set(ActiveEnrichment.class, found);
+ }
+ this.state = found;
+ this.wasSuspended = found.suspended;
+ this.wereRecognizersSuspended = found.recognizersSuspended;
+ if (recognizersOnly) {
+ found.recognizersSuspended = true;
+ } else {
+ found.suspended = true;
+ }
+ }
+
+ @Override
+ public void close() {
+ state.suspended = wasSuspended;
+ state.recognizersSuspended = wereRecognizersSuspended;
+ }
+ }
+
private static Parser unwrap(Parser parser) {
while (parser instanceof ParserDecorator decorator) {
parser = decorator.getWrappedParser();
@@ -418,9 +501,11 @@ public final class ContentEnrichers {
}
}
- /** Mutable per-parse marker; single-threaded within one parse. */
+ /** Mutable per-parse dispatch state; single-threaded within one parse. */
static final class ActiveEnrichment {
boolean active;
+ boolean suspended;
+ boolean recognizersSuspended;
}
/**
diff --git
a/tika-core/src/test/java/org/apache/tika/parser/enricher/ContentEnrichersTest.java
b/tika-core/src/test/java/org/apache/tika/parser/enricher/ContentEnrichersTest.java
index 975f9964de..98f0cf109c 100644
---
a/tika-core/src/test/java/org/apache/tika/parser/enricher/ContentEnrichersTest.java
+++
b/tika-core/src/test/java/org/apache/tika/parser/enricher/ContentEnrichersTest.java
@@ -149,6 +149,66 @@ public class ContentEnrichersTest {
assertFalse(ContentEnrichers.hasTextRecognizer(listOf(annotator), PNG,
target(PNG), context));
}
+ @Test
+ public void testSuspendedRecognizers() throws Exception {
+ ParseContext context = new ParseContext();
+ RecordingParser annotator = new RecordingParser(Set.of(PNG));
+ RecognizingParser recognizer = new RecognizingParser(Set.of(PNG),
true);
+ RecognizingParser declining = new RecognizingParser(Set.of(PNG),
false);
+ RecordingParser legacy = new RecordingParser(Set.of(OCR_PNG));
+ CompositeContentEnricher list = listOf(recognizer, annotator,
declining, legacy);
+
+ Parser annotators;
+ try (ContentEnrichers.Suspension scope =
ContentEnrichers.suspendRecognizers(context)) {
+ annotators = ContentEnrichers.get(list, PNG, target(PNG), context);
+ assertNotNull(annotators);
+ assertFalse(ContentEnrichers.hasTextRecognizer(list, PNG,
target(PNG), context));
+ assertNull(ContentEnrichers.get(listOf(recognizer), PNG,
target(PNG), context));
+ }
+ invoke(annotators, new Metadata(), context);
+ assertEquals(1, annotator.calls);
+ assertEquals(0, recognizer.calls);
+ assertEquals(0, declining.calls, "a recognizer that declines is not an
annotator");
+ assertEquals(0, legacy.calls, "a legacy claimant is a recognizer");
+ assertNotNull(ContentEnrichers.get(listOf(recognizer), PNG,
target(PNG), context));
+ assertTrue(ContentEnrichers.hasTextRecognizer(list, PNG, target(PNG),
context));
+
+ // no list: the discovered engine is filtered the same way
+ context.set(Parser.class, compositeOf(new
AnnotatingParser(Set.of(PNG))));
+ try (ContentEnrichers.Suspension scope =
ContentEnrichers.suspendRecognizers(context)) {
+ assertNotNull(ContentEnrichers.get(null, PNG, target(PNG),
context));
+ }
+ context.set(Parser.class, compositeOf(new
RecognizingParser(Set.of(PNG), true)));
+ try (ContentEnrichers.Suspension scope =
ContentEnrichers.suspendRecognizers(context)) {
+ assertNull(ContentEnrichers.get(null, PNG, target(PNG), context));
+ }
+ assertNotNull(ContentEnrichers.get(null, PNG, target(PNG), context));
+ }
+
+ @Test
+ public void testSuspendedDispatch() throws Exception {
+ ParseContext context = new ParseContext();
+ CompositeContentEnricher list = listOf(new
RecognizingParser(Set.of(PNG), true),
+ new RecordingParser(Set.of(PNG)));
+ try (ContentEnrichers.Suspension outer =
ContentEnrichers.suspend(context)) {
+ assertNull(ContentEnrichers.get(list, PNG, target(PNG), context));
+ assertFalse(ContentEnrichers.hasTextRecognizer(list, PNG,
target(PNG), context));
+ try (ContentEnrichers.Suspension inner =
ContentEnrichers.suspend(context)) {
+ assertNull(ContentEnrichers.get(list, PNG, target(PNG),
context));
+ }
+ assertNull(ContentEnrichers.get(list, PNG, target(PNG), context),
+ "an inner scope restores the outer suspension, it does not
lift it");
+ try (ContentEnrichers.Suspension inner =
+ ContentEnrichers.suspendRecognizers(context)) {
+ assertNull(ContentEnrichers.get(list, PNG, target(PNG),
context));
+ }
+ assertNull(ContentEnrichers.get(list, PNG, target(PNG), context),
+ "a recognizer scope inside a full suspension changes
nothing on close");
+ }
+ assertNotNull(ContentEnrichers.get(list, PNG, target(PNG), context));
+ assertTrue(ContentEnrichers.hasTextRecognizer(list, PNG, target(PNG),
context));
+ }
+
@Test
public void testHasTextRecognizerRefusedDuringEnrichment() throws
Exception {
ParseContext context = new ParseContext();
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
index 8bb72323d7..f8e41ba0c2 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
@@ -170,6 +170,11 @@ class AbstractPDF2XHTML extends PDFTextStripper {
private final PageTextBuffer pageBuffer;
// Every enricher for the render type, resolved once per document; null
skips the render.
final Parser ocrEngine;
+
+ // Non-null only when the image strategy renders every page: the enrichers
that do not
+ // recognize text run on each page the OCR step leaves alone, so a page is
enriched once
+ // whether or not it tripped the OCR verdict.
+ private final Parser pageAnnotators;
final MediaType ocrImageMediaType;
private PageText pageDecision = PageText.UNDECIDED;
// Text held back from an OCR_WANTED page; replayed if OCR does not run.
@@ -228,6 +233,17 @@ class AbstractPDF2XHTML extends PDFTextStripper {
TikaCoreProperties.EmbeddedResourceType.RENDERING.name());
this.ocrEngine =
ContentEnrichers.get(contentEnrichers, ocrImageMediaType,
renderTarget, context);
+ PDFParserConfig.IMAGE_STRATEGY imageStrategy =
config.getImageStrategy();
+ Parser annotators = null;
+ if (imageStrategy ==
PDFParserConfig.IMAGE_STRATEGY.RENDER_PAGES_BEFORE_PARSE
+ || imageStrategy ==
PDFParserConfig.IMAGE_STRATEGY.RENDER_PAGES_AT_PAGE_END) {
+ try (ContentEnrichers.Suspension recognizersOff =
+ ContentEnrichers.suspendRecognizers(context)) {
+ annotators = ContentEnrichers.get(contentEnrichers,
ocrImageMediaType,
+ renderTarget, context);
+ }
+ }
+ this.pageAnnotators = annotators;
if (config.getOcr().getStrategy() == AUTO &&
ContentEnrichers.hasTextRecognizer(
contentEnrichers, ocrImageMediaType, renderTarget, context)) {
this.pageBuffer = new PageTextBuffer(handler);
@@ -637,15 +653,19 @@ class AbstractPDF2XHTML extends PDFTextStripper {
}
}
- /**
- * @return true if the engine wrote text for this page; false when there
is no engine
- * (AUTO), maxPagesToOcr is exhausted, a failure was recorded and
swallowed, or the
- * engine produced no text (an annotating enricher, or one told to skip
OCR)
- */
- boolean doOCROnCurrentPage(PDPage pdPage, OcrConfig.Strategy ocrStrategy)
+ /** What the OCR step did with a page. */
+ enum PageOcr {
+ /** No engine was dispatched: NO_OCR, no engine, or maxPagesToOcr
spent. */
+ SKIPPED,
+ /** The engine ran but wrote no text: an annotating enricher, a skip,
a failure. */
+ NO_TEXT,
+ TEXT
+ }
+
+ PageOcr doOCROnCurrentPage(PDPage pdPage, OcrConfig.Strategy ocrStrategy)
throws IOException, TikaException, SAXException {
if (ocrStrategy.equals(NO_OCR)) {
- return false;
+ return PageOcr.SKIPPED;
}
//count the number of times that OCR would have been called
OCRPageCounter c = context.get(OCRPageCounter.class);
@@ -656,7 +676,7 @@ class AbstractPDF2XHTML extends PDFTextStripper {
// Enforce maxPagesToOcr limit
int maxPagesToOcr = config.getOcr().getMaxPagesToOcr();
if (maxPagesToOcr > 0 && c != null && c.getCount() > maxPagesToOcr) {
- return false;
+ return PageOcr.SKIPPED;
}
if (ocrEngine == null) {
if (ocrStrategy == OCR_ONLY || ocrStrategy ==
OCR_AND_TEXT_EXTRACTION) {
@@ -667,16 +687,31 @@ class AbstractPDF2XHTML extends PDFTextStripper {
"add one to the classpath when no list is
configured, " +
"or set the OCR strategy to NO_OCR.");
}
- return false;
+ return PageOcr.SKIPPED;
+ }
+ return enrichCurrentPage(pdPage, ocrEngine) ? PageOcr.TEXT :
PageOcr.NO_TEXT;
+ }
+
+ /**
+ * Runs the annotating enrichers on a page the OCR step skipped, when
every page is
+ * rendered. The rendering emitted as an embedded document is not enriched
again.
+ */
+ void annotateCurrentPage(PDPage pdPage) throws IOException, TikaException,
SAXException {
+ if (pageAnnotators != null) {
+ enrichCurrentPage(pdPage, pageAnnotators);
}
+ }
+ /** Renders the page and runs the engine on it; true if the engine wrote
text. */
+ private boolean enrichCurrentPage(PDPage pdPage, Parser engine)
+ throws IOException, TikaException, SAXException {
try (TemporaryResources tmp = new TemporaryResources()) {
try (RenderResult renderResult = renderCurrentPage(pdPage, tmp)) {
Metadata renderMetadata = renderResult.getMetadata();
TextCounter counter = new TextCounter(xhtml);
try (TikaInputStream tis = renderResult.getInputStream()) {
renderMetadata.set(HttpHeaders.CONTENT_TYPE,
ocrImageMediaType.toString());
- ocrEngine.parse(tis,
+ engine.parse(tis,
new EmbeddedContentHandler(new
BodyContentHandler(counter)),
renderMetadata, context);
}
@@ -878,11 +913,17 @@ class AbstractPDF2XHTML extends PDFTextStripper {
for (PDAnnotation annotation : page.getAnnotations()) {
processPageAnnotation(annotation);
}
+ PageOcr ocr = PageOcr.SKIPPED;
if (config.getOcr().getStrategy() == OCR_AND_TEXT_EXTRACTION) {
- doOCROnCurrentPage(page, OCR_AND_TEXT_EXTRACTION);
- } else if (pageDecision == PageText.OCR_WANTED &&
- !doOCROnCurrentPage(page, AUTO)) {
- replay(pendingText);
+ ocr = doOCROnCurrentPage(page, OCR_AND_TEXT_EXTRACTION);
+ } else if (pageDecision == PageText.OCR_WANTED) {
+ ocr = doOCROnCurrentPage(page, AUTO);
+ if (ocr != PageOcr.TEXT) {
+ replay(pendingText);
+ }
+ }
+ if (ocr == PageOcr.SKIPPED) {
+ annotateCurrentPage(page);
}
PDPageAdditionalActions pageActions = page.getActions();
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/OCR2XHTML.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/OCR2XHTML.java
index 3e8a2bd70d..ae23a5e6e2 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/OCR2XHTML.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/OCR2XHTML.java
@@ -97,7 +97,9 @@ class OCR2XHTML extends AbstractPDF2XHTML {
public void processPage(PDPage pdPage) throws IOException {
try {
startPage(pdPage);
- doOCROnCurrentPage(pdPage, OcrConfig.Strategy.OCR_ONLY);
+ if (doOCROnCurrentPage(pdPage, OcrConfig.Strategy.OCR_ONLY) ==
PageOcr.SKIPPED) {
+ annotateCurrentPage(pdPage);
+ }
endPage(pdPage);
} catch (TikaException | SAXException e) {
throw new IOException(e);
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java
index 4be844c909..e23db89cc0 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java
@@ -43,6 +43,7 @@ import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.enricher.CompositeContentEnricher;
+import org.apache.tika.parser.enricher.ContentEnrichers;
import org.apache.tika.parser.pdf.image.ImageGraphicsEngine;
import org.apache.tika.renderer.PageRangeRequest;
import org.apache.tika.renderer.RenderRequest;
@@ -174,7 +175,9 @@ class PDF2XHTML extends AbstractPDF2XHTML {
RenderRequest request = new PageRangeRequest(getCurrentPageNo(),
getCurrentPageNo());
Metadata renderedMetadata = Metadata.newInstance(context);
renderedMetadata.set(TikaCoreProperties.TYPE,
PDFParser.MEDIA_TYPE.toString());
- try (RenderResults results = renderer.render(tis, renderedMetadata,
context, request)) {
+ // the page step enriches the render itself; the embedded copy is
bytes and metadata
+ try (RenderResults results = renderer.render(tis, renderedMetadata,
context, request);
+ ContentEnrichers.Suspension suspension =
ContentEnrichers.suspend(context)) {
for (RenderResult result : results.getResults()) {
if (result.getStatus() == RenderResult.STATUS.SUCCESS) {
if
(embeddedDocumentExtractor.shouldParseEmbedded(result.getMetadata(), context)) {
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
index 5c987e1e64..97a277eef2 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
@@ -80,6 +80,7 @@ import org.apache.tika.parser.Parser;
import org.apache.tika.parser.PasswordProvider;
import org.apache.tika.parser.RenderingParser;
import org.apache.tika.parser.enricher.CompositeContentEnricher;
+import org.apache.tika.parser.enricher.ContentEnrichers;
import org.apache.tika.parser.enricher.EnrichingParser;
import org.apache.tika.parser.pdf.updates.IncrementalUpdateRecord;
import org.apache.tika.parser.pdf.updates.IsIncrementalUpdate;
@@ -467,15 +468,18 @@ public class PDFParser implements Parser,
RenderingParser, EnrichingParser {
EmbeddedDocumentExtractor embeddedDocumentExtractor =
EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(context);
- for (RenderResult result : renderResults.getResults()) {
- if (result.getStatus() == RenderResult.STATUS.SUCCESS) {
- if
(embeddedDocumentExtractor.shouldParseEmbedded(result.getMetadata(), context)) {
- try (TikaInputStream tis = result.getInputStream()) {
- embeddedDocumentExtractor.parseEmbedded(tis, xhtml,
result.getMetadata(), context, false);
- } catch (SecurityException e) {
- throw e;
- } catch (Exception e) {
- EmbeddedDocumentUtil.recordException(e,
parentMetadata, context);
+ // the page step enriches each render itself; the embedded copies are
bytes and metadata
+ try (ContentEnrichers.Suspension suspension =
ContentEnrichers.suspend(context)) {
+ for (RenderResult result : renderResults.getResults()) {
+ if (result.getStatus() == RenderResult.STATUS.SUCCESS) {
+ if
(embeddedDocumentExtractor.shouldParseEmbedded(result.getMetadata(), context)) {
+ try (TikaInputStream tis = result.getInputStream()) {
+ embeddedDocumentExtractor.parseEmbedded(tis,
xhtml, result.getMetadata(), context, false);
+ } catch (SecurityException e) {
+ throw e;
+ } catch (Exception e) {
+ EmbeddedDocumentUtil.recordException(e,
parentMetadata, context);
+ }
}
}
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
index 75877f8302..2852ce22cd 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
@@ -78,6 +78,8 @@ import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.PasswordProvider;
import org.apache.tika.parser.enricher.CompositeContentEnricher;
+import org.apache.tika.parser.enricher.ContentEnricher;
+import org.apache.tika.parser.enricher.ContentEnrichers;
import org.apache.tika.parser.enricher.TextRecognizer;
import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.sax.ContentHandlerDecorator;
@@ -1721,6 +1723,153 @@ public class PDFParserTest extends TikaTest {
assertWellFormed(xml);
}
+ /**
+ * Pages rendered as embedded documents are enriched once, by the PDF
parser: the text
+ * recognizer where the OCR strategy says so, every other enricher on
every page. The
+ * embedded copy of a render is parsed but not enriched again.
+ */
+ @Test
+ public void testRenderedPagesEnrichedOnceByPdfParser() throws Exception {
+ for (PDFParserConfig.IMAGE_STRATEGY imageStrategy : List.of(
+ PDFParserConfig.IMAGE_STRATEGY.RENDER_PAGES_AT_PAGE_END,
+ PDFParserConfig.IMAGE_STRATEGY.RENDER_PAGES_BEFORE_PARSE)) {
+ PageEnricher recognizer = new PageEnricher("MOCK_OCR_CONTENT",
true);
+ PageAnnotator annotator = new PageAnnotator();
+ CompositeContentEnricher enrichers =
+ new CompositeContentEnricher(List.of(recognizer,
annotator));
+ PDFParser parser = new PDFParser();
+ parser.setContentEnrichers(enrichers);
+ EnrichingImageParser imageParser = new
EnrichingImageParser(enrichers);
+ PDFParserConfig config = autoOcrTriggeringPage16();
+ config.setImageStrategy(imageStrategy);
+ config.getOcr().setDpi(20);
+ ParseContext context = new ParseContext();
+ context.set(PDFParserConfig.class, config);
+ context.set(Parser.class, new AutoDetectParser(imageParser));
+ Metadata metadata = new Metadata();
+
+ String xml = parsePdfToXml(parser, "testPDF_bad_page_303226.pdf",
metadata, context);
+
+ String label = imageStrategy.name();
+ assertEquals(19, imageParser.parsed, label + ": every render is an
embedded document");
+ assertEquals(1, recognizer.calls, label + ": the recognizer sees
the verdict page only");
+ assertEquals(19, annotator.calls, label + ": the annotator sees
each page once");
+ assertContainsCount("MOCK_OCR_CONTENT", xml, 1);
+ assertNotContained("42936", xml);
+ assertContains("ANABOLIC", xml);
+ assertEquals(1, metadata.getInt(PDF.OCR_PAGE_COUNT));
+ assertEquals(19,
metadata.get(TikaCoreProperties.TIKA_CHUNKS).split("\"page\":").length - 1,
+ label + ": one chunk per page on the PDF itself");
+ }
+ }
+
+ /** NO_OCR governs recognizers only: rendered pages are still annotated,
once. */
+ @Test
+ public void testNoOcrStillAnnotatesRenderedPages() throws Exception {
+ PageEnricher recognizer = new PageEnricher("MOCK_OCR_CONTENT", true);
+ PageAnnotator annotator = new PageAnnotator();
+ CompositeContentEnricher enrichers =
+ new CompositeContentEnricher(List.of(recognizer, annotator));
+ PDFParser parser = new PDFParser();
+ parser.setContentEnrichers(enrichers);
+ EnrichingImageParser imageParser = new EnrichingImageParser(enrichers);
+ PDFParserConfig config = new PDFParserConfig();
+ config.getOcr().setStrategy(OcrConfig.Strategy.NO_OCR);
+
config.setImageStrategy(PDFParserConfig.IMAGE_STRATEGY.RENDER_PAGES_AT_PAGE_END);
+ config.getOcr().setDpi(20);
+ ParseContext context = new ParseContext();
+ context.set(PDFParserConfig.class, config);
+ context.set(Parser.class, new AutoDetectParser(imageParser));
+ Metadata metadata = new Metadata();
+
+ String xml = parsePdfToXml(parser, "testPDF_bookmarks.pdf", metadata,
context);
+
+ assertEquals(2, imageParser.parsed);
+ assertEquals(0, recognizer.calls);
+ assertEquals(2, annotator.calls);
+ assertNotContained("MOCK_OCR_CONTENT", xml);
+ assertEquals(0, metadata.getInt(PDF.OCR_PAGE_COUNT));
+ }
+
+ /** maxPagesToOcr budgets the recognizer; the annotator still sees every
rendered page. */
+ @Test
+ public void testOcrBudgetDoesNotBudgetAnnotators() throws Exception {
+ PageEnricher recognizer = new PageEnricher("MOCK_OCR_CONTENT", true);
+ PageAnnotator annotator = new PageAnnotator();
+ CompositeContentEnricher enrichers =
+ new CompositeContentEnricher(List.of(recognizer, annotator));
+ PDFParser parser = new PDFParser();
+ parser.setContentEnrichers(enrichers);
+ PDFParserConfig config = new PDFParserConfig();
+
config.getOcr().setStrategy(OcrConfig.Strategy.OCR_AND_TEXT_EXTRACTION);
+ config.getOcr().setMaxPagesToOcr(1);
+
config.setImageStrategy(PDFParserConfig.IMAGE_STRATEGY.RENDER_PAGES_AT_PAGE_END);
+ config.getOcr().setDpi(20);
+ ParseContext context = new ParseContext();
+ context.set(PDFParserConfig.class, config);
+ context.set(Parser.class, new AutoDetectParser(new
EnrichingImageParser(enrichers)));
+ Metadata metadata = new Metadata();
+
+ String xml = parsePdfToXml(parser, "testPDF_bookmarks.pdf", metadata,
context);
+
+ assertEquals(1, recognizer.calls);
+ assertEquals(2, annotator.calls);
+ assertContainsCount("MOCK_OCR_CONTENT", xml, 1);
+ }
+
+ /** Annotating enricher for image/png: writes one chunk per render, never
text. */
+ private static class PageAnnotator implements Parser, ContentEnricher {
+ private static final long serialVersionUID = 1L;
+ int calls = 0;
+
+ @Override
+ public Set<MediaType> getSupportedTypes(ParseContext context) {
+ return Collections.singleton(MediaType.image("png"));
+ }
+
+ @Override
+ public void parse(TikaInputStream tis, ContentHandler handler,
Metadata metadata,
+ ParseContext context) throws IOException,
SAXException, TikaException {
+ calls++;
+ metadata.setTrusted(TikaCoreProperties.TIKA_CHUNKS.getName(),
+ "[{\"page\":" + metadata.get(TikaPagedText.PAGE_NUMBER) +
"}]");
+ XHTMLContentHandler xhtml = new XHTMLContentHandler(handler,
metadata);
+ xhtml.startDocument();
+ xhtml.endDocument();
+ }
+ }
+
+ /** Stands in for image-parser: dispatches image/png to the content
enrichers. */
+ private static class EnrichingImageParser implements Parser {
+ private static final long serialVersionUID = 1L;
+ private final CompositeContentEnricher enrichers;
+ int parsed = 0;
+
+ EnrichingImageParser(CompositeContentEnricher enrichers) {
+ this.enrichers = enrichers;
+ }
+
+ @Override
+ public Set<MediaType> getSupportedTypes(ParseContext context) {
+ return Collections.singleton(MediaType.image("png"));
+ }
+
+ @Override
+ public void parse(TikaInputStream tis, ContentHandler handler,
Metadata metadata,
+ ParseContext context) throws IOException,
SAXException, TikaException {
+ parsed++;
+ Parser enricher =
+ ContentEnrichers.get(enrichers, MediaType.image("png"),
metadata, context);
+ if (enricher != null) {
+ enricher.parse(tis, handler, metadata, context);
+ return;
+ }
+ XHTMLContentHandler xhtml = new XHTMLContentHandler(handler,
metadata);
+ xhtml.startDocument();
+ xhtml.endDocument();
+ }
+ }
+
/** Enricher for image/png that writes the given text (none if null). */
private static class PageEnricher implements Parser, TextRecognizer {
private static final long serialVersionUID = 1L;
