This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new d8fe4bc7ac update ocr configs for PDFs (#3180)
d8fe4bc7ac is described below
commit d8fe4bc7ac3784141cdd1c9cf07f634e5cf4d969
Author: Tim Allison <[email protected]>
AuthorDate: Tue Sep 15 15:04:56 2026 -0400
update ocr configs for PDFs (#3180)
---
CHANGES.txt | 9 +
.../pages/configuration/parsers/pdf-parser.adoc | 23 ++
.../resources/config-examples/pdf-parser-full.json | 12 +
.../apache/tika/parser/pdf/AbstractPDF2XHTML.java | 11 +-
.../java/org/apache/tika/parser/pdf/PDF2XHTML.java | 29 +-
.../java/org/apache/tika/parser/pdf/PDFParser.java | 34 ++-
.../apache/tika/parser/pdf/PDFParserConfig.java | 10 +
.../apache/tika/parser/pdf/RenderingConfig.java | 132 +++++++++
.../tika/renderer/pdf/pdfbox/PDFBoxRenderer.java | 72 +++--
.../tika/parser/pdf/PDFRenderingConfigTest.java | 303 +++++++++++++++++++++
10 files changed, 586 insertions(+), 49 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 27a3d6d8af..bf78778206 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,14 @@
Release 4.1.0 - unreleased
+ * PDF: "pdf-parser": {"rendering": {"dpi", "imageType", "imageFormat",
+ "imageQuality", "maxImagePixels"}} sets the page images emitted under
+ RENDER_PAGES_BEFORE_PARSE and RENDER_PAGES_AT_PAGE_END; a field left out
+ takes the "ocr" value. OCR, annotators and PAGES inference input keep
+ rendering with "ocr", and share the emitted render only when it is the
+ same image. maxImagePixels now also bounds pages rendered through
+ pdfbox-renderer; a page it skips leaves a warning on the PDF instead of
+ vanishing (TIKA-4889).
+
* Tagged PDFs: text drawn twice for a bold effect with one copy tagged
came out as letters with spaces between them; each copy now reads as
its words. A tree that puts every glyph in its own paragraph (some form
diff --git a/docs/modules/ROOT/pages/configuration/parsers/pdf-parser.adoc
b/docs/modules/ROOT/pages/configuration/parsers/pdf-parser.adoc
index 039beacf62..5a1714f04c 100644
--- a/docs/modules/ROOT/pages/configuration/parsers/pdf-parser.adoc
+++ b/docs/modules/ROOT/pages/configuration/parsers/pdf-parser.adoc
@@ -95,6 +95,29 @@ twice, and `EXTRACT` means no page OCR anywhere while the
other enrichers still
`pdf:ocr-page-count` records how many pages tripped the verdict, whether or
not an engine ran.
+=== Page images (`rendering`)
+
+The page images the parser emits as embedded documents under
`RENDER_PAGES_BEFORE_PARSE` or
+`RENDER_PAGES_AT_PAGE_END` have their own `rendering` block: `dpi`,
`imageType`, `imageFormat`,
+`imageQuality` and `maxImagePixels`. A field left out takes the `ocr` value,
so with no block
+the emitted pages match the OCR renders as in 4.0. OCR, page annotators and
PAGES inference
+input always render with the `ocr` settings; when the two blocks would produce
the same image,
+one render per page serves both, otherwise OCR renders again.
+
+[source,json]
+----
+{
+ "pdf-parser": {
+ "imageStrategy": "RENDER_PAGES_AT_PAGE_END",
+ "maxRenderedPages": 1,
+ "rendering": { "dpi": 96, "imageType": "RGB" }
+ }
+}
+----
+
+renders a colour first-page preview at 96 dpi while OCR keeps its 300 dpi
grayscale pages. A
+page that would exceed `maxImagePixels` is not rendered and leaves a warning
on the PDF.
+
== Tagged PDFs (`markedContent`)
A tagged PDF carries a structure tree: paragraphs, headings, lists, tables,
links, and which
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/resources/config-examples/pdf-parser-full.json
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/resources/config-examples/pdf-parser-full.json
index 2e1ae9e0d3..b3619743f6 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/resources/config-examples/pdf-parser-full.json
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/resources/config-examples/pdf-parser-full.json
@@ -103,6 +103,18 @@
},
// Parse prior incremental-update versions as embedded documents.
"parseIncrementalUpdates": false,
+ // The page images emitted under RENDER_PAGES_BEFORE_PARSE /
RENDER_PAGES_AT_PAGE_END.
+ // A field left out takes the "ocr" value; OCR itself always renders
with "ocr".
+ "rendering": {
+ "dpi": 96,
+ // Options: RGB, GRAY
+ "imageType": "RGB",
+ // Options: PNG, TIFF, JPEG
+ "imageFormat": "PNG",
+ "imageQuality": 0.5,
+ // Skip pages that would render larger than this area (w x h); -1 =
no limit.
+ "maxImagePixels": 100000000
+ },
// EXPERT: set the Sun KCMS color-management system property. Default
false.
"setKCMS": false,
// Sort text by x/y position; helps some PDFs, can interleave columns
in others.
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
index 835192d828..5fa8ed36c1 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
@@ -842,11 +842,8 @@ class AbstractPDF2XHTML extends PDFTextStripper {
//if the full document has already been rendered, then reuse that file
//TODO: we need to prevent this if only a portion of the page or
portions
//of the page have been rendered.
- //TODO: we should also figure out how to not reuse the rendering if
- //the user wants to render twice (say, full color to display to users,
but
- //grayscale for (notionally?) better OCR).
PageBasedRenderResults results = (PageBasedRenderResults)
renderingState.getRenderResults();
- if (results != null) {
+ if (results != null && pageImageIsOcrImage()) {
List<RenderResult> pageResults =
results.getPage(getCurrentPageNo());
if (pageResults.size() == 1) {
return pageResults.get(0);
@@ -879,6 +876,12 @@ class AbstractPDF2XHTML extends PDFTextStripper {
}
}
+ /** A page image rendered before the parse serves OCR only if OCR would
render it the same. */
+ private boolean pageImageIsOcrImage() {
+ return config.getRendering().resolve(config.getOcr())
+ .rendersSameImageAs(RenderingConfig.from(config.getOcr()));
+ }
+
private Renderer getPDFRenderer() {
if (renderer == null) {
return null;
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java
index 033619eddd..7663c7d28b 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDF2XHTML.java
@@ -164,6 +164,18 @@ class PDF2XHTML extends AbstractPDF2XHTML {
}
}
+ /** The page image for the consumer: rendered with the "rendering"
settings, not OCR's. */
+ private RenderResults renderPageImage(TikaInputStream tis, Metadata
renderedMetadata,
+ RenderRequest request) throws
IOException, TikaException {
+ RenderingConfig outer = context.get(RenderingConfig.class);
+ context.set(RenderingConfig.class,
config.getRendering().resolve(config.getOcr()));
+ try {
+ return renderer.render(tis, renderedMetadata, context, request);
+ } finally {
+ context.set(RenderingConfig.class, outer);
+ }
+ }
+
private void renderPage(PDPage page) throws IOException {
if (config.getImageStrategy() !=
PDFParserConfig.IMAGE_STRATEGY.RENDER_PAGES_AT_PAGE_END) {
return;
@@ -181,17 +193,16 @@ class PDF2XHTML extends AbstractPDF2XHTML {
Metadata renderedMetadata = Metadata.newInstance(context);
renderedMetadata.set(TikaCoreProperties.TYPE,
PDFParser.MEDIA_TYPE.toString());
// the page step enriches the render itself; the embedded copy is
bytes and metadata
- try (RenderResults results = renderer.render(tis, renderedMetadata,
context, request);
+ try (RenderResults results = renderPageImage(tis, renderedMetadata,
request);
ContentEnrichers.Suspension suspension =
ContentEnrichers.suspend(context)) {
for (RenderResult result : results.getResults()) {
- if (result.getStatus() == RenderResult.STATUS.SUCCESS) {
- if
(embeddedDocumentExtractor.shouldParseEmbedded(result.getMetadata(), context)) {
-
- try (TikaInputStream resultInputStream =
result.getInputStream()) {
- //TODO: add markup here?
-
embeddedDocumentExtractor.parseEmbedded(resultInputStream, xhtml,
- result.getMetadata(), context, true);
- }
+ if (result.getStatus() != RenderResult.STATUS.SUCCESS) {
+ PDFParser.carryRenderWarnings(result, metadata);
+ } else if
(embeddedDocumentExtractor.shouldParseEmbedded(result.getMetadata(), context)) {
+ try (TikaInputStream resultInputStream =
result.getInputStream()) {
+ //TODO: add markup here?
+
embeddedDocumentExtractor.parseEmbedded(resultInputStream, xhtml,
+ result.getMetadata(), context, true);
}
}
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
index 4572bd48d5..ab2f6ca499 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
@@ -474,21 +474,29 @@ public class PDFParser implements Parser,
RenderingParser, EnrichingParser {
// the page step enriches each render itself; the embedded copies are
bytes and metadata
try (ContentEnrichers.Suspension suspension =
ContentEnrichers.suspend(context)) {
for (RenderResult result : renderResults.getResults()) {
- if (result.getStatus() == RenderResult.STATUS.SUCCESS) {
- if
(embeddedDocumentExtractor.shouldParseEmbedded(result.getMetadata(), context)) {
- try (TikaInputStream tis = result.getInputStream()) {
- embeddedDocumentExtractor.parseEmbedded(tis,
xhtml, result.getMetadata(), context, false);
- } catch (SecurityException e) {
- throw e;
- } catch (Exception e) {
- EmbeddedDocumentUtil.recordException(e,
parentMetadata, context);
- }
+ if (result.getStatus() != RenderResult.STATUS.SUCCESS) {
+ carryRenderWarnings(result, parentMetadata);
+ } else if
(embeddedDocumentExtractor.shouldParseEmbedded(result.getMetadata(), context)) {
+ try (TikaInputStream tis = result.getInputStream()) {
+ embeddedDocumentExtractor.parseEmbedded(tis, xhtml,
result.getMetadata(), context, false);
+ } catch (SecurityException e) {
+ throw e;
+ } catch (Exception e) {
+ EmbeddedDocumentUtil.recordException(e,
parentMetadata, context);
}
}
}
}
}
+ /** A page the renderer could not make is a warning on the PDF, not a
silent gap. */
+ static void carryRenderWarnings(RenderResult result, Metadata
parentMetadata) {
+ for (String warning : result.getMetadata()
+ .getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING)) {
+ parentMetadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
warning);
+ }
+ }
+
private RenderResults renderPDF(TikaInputStream tstream,
ParseContext parseContext, PDFParserConfig
localConfig)
throws IOException, TikaException {
@@ -497,7 +505,13 @@ public class PDFParser implements Parser, RenderingParser,
EnrichingParser {
int maxRenderedPages = localConfig.getMaxRenderedPages();
PageRangeRequest pages = maxRenderedPages > 0
? new PageRangeRequest(1, maxRenderedPages) :
PageRangeRequest.RENDER_ALL;
- return renderer.render(tstream, metadata, parseContext, pages);
+ RenderingConfig outer = parseContext.get(RenderingConfig.class);
+ parseContext.set(RenderingConfig.class,
localConfig.getRendering().resolve(localConfig.getOcr()));
+ try {
+ return renderer.render(tstream, metadata, parseContext, pages);
+ } finally {
+ parseContext.set(RenderingConfig.class, outer);
+ }
}
protected PDDocument getPDDocument(TikaInputStream tis, String password,
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
index d7948bef97..0d53a3b3e5 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
@@ -135,6 +135,7 @@ public class PDFParserConfig implements Serializable {
private boolean ifXFAExtractOnlyXFA = false;
private OcrConfig ocr = new OcrConfig();
+ private RenderingConfig rendering = new RenderingConfig();
/** Null until set; {@link #getText()} then falls back to the {@code
ocr.strategy} alias. */
private TextPolicy text;
@@ -552,6 +553,15 @@ public class PDFParserConfig implements Serializable {
this.ocr = ocr;
}
+ /** Settings for the page images emitted under RENDER_PAGES_*; unset
fields follow {@code ocr}. */
+ public RenderingConfig getRendering() {
+ return rendering;
+ }
+
+ public void setRendering(RenderingConfig rendering) {
+ this.rendering = rendering == null ? new RenderingConfig() : rendering;
+ }
+
/** The text policy: {@code "text"} if set, else the {@code ocr.strategy}
alias, else AUTO. */
public TextPolicy getText() {
if (text != null) {
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/RenderingConfig.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/RenderingConfig.java
new file mode 100644
index 0000000000..8b76a20543
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/RenderingConfig.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.pdf;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * Settings for the page images the parser emits as embedded documents under
+ * {@code imageStrategy} {@code RENDER_PAGES_BEFORE_PARSE} or {@code
RENDER_PAGES_AT_PAGE_END}:
+ * {@code "pdf-parser": {"rendering": {"dpi": 96, "imageType": "RGB"}}}. A
field left unset
+ * takes the {@code ocr} value, so with no block at all these renders keep
matching the OCR
+ * renders as they did in 4.0. OCR, page annotators and PAGES inference input
always render
+ * with the {@code ocr} settings.
+ *
+ * @since Apache Tika 4.1.0
+ */
+public class RenderingConfig implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private Integer dpi;
+ private OcrConfig.ImageType imageType;
+ private OcrConfig.ImageFormat imageFormat;
+ private Float imageQuality;
+ private Long maxImagePixels;
+
+ /** The OCR settings, as a fully set block. */
+ public static RenderingConfig from(OcrConfig ocr) {
+ RenderingConfig config = new RenderingConfig();
+ config.dpi = ocr.getDpi();
+ config.imageType = ocr.getImageType();
+ config.imageFormat = ocr.getImageFormat();
+ config.imageQuality = ocr.getImageQuality();
+ config.maxImagePixels = ocr.getMaxImagePixels();
+ return config;
+ }
+
+ /** This block with every unset field taken from {@code ocr}. */
+ public RenderingConfig resolve(OcrConfig ocr) {
+ RenderingConfig resolved = from(ocr);
+ if (dpi != null) {
+ resolved.dpi = dpi;
+ }
+ if (imageType != null) {
+ resolved.imageType = imageType;
+ }
+ if (imageFormat != null) {
+ resolved.imageFormat = imageFormat;
+ }
+ if (imageQuality != null) {
+ resolved.imageQuality = imageQuality;
+ }
+ if (maxImagePixels != null) {
+ resolved.maxImagePixels = maxImagePixels;
+ }
+ return resolved;
+ }
+
+ /** True when a page rendered with either block is the same image. */
+ public boolean rendersSameImageAs(RenderingConfig other) {
+ return Objects.equals(dpi, other.dpi) && imageType == other.imageType
+ && imageFormat == other.imageFormat
+ && Objects.equals(imageQuality, other.imageQuality);
+ }
+
+ public Integer getDpi() {
+ return dpi;
+ }
+
+ public void setDpi(Integer dpi) {
+ if (dpi != null && dpi < 1) {
+ throw new IllegalArgumentException("dpi must be at least 1, got: "
+ dpi);
+ }
+ this.dpi = dpi;
+ }
+
+ public OcrConfig.ImageType getImageType() {
+ return imageType;
+ }
+
+ public void setImageType(OcrConfig.ImageType imageType) {
+ this.imageType = imageType;
+ }
+
+ public OcrConfig.ImageFormat getImageFormat() {
+ return imageFormat;
+ }
+
+ public void setImageFormat(OcrConfig.ImageFormat imageFormat) {
+ this.imageFormat = imageFormat;
+ }
+
+ public Float getImageQuality() {
+ return imageQuality;
+ }
+
+ public void setImageQuality(Float imageQuality) {
+ if (imageQuality != null && (imageQuality < 0f || imageQuality > 1f)) {
+ throw new IllegalArgumentException(
+ "imageQuality must be between 0.0 and 1.0, got: " +
imageQuality);
+ }
+ this.imageQuality = imageQuality;
+ }
+
+ public Long getMaxImagePixels() {
+ return maxImagePixels;
+ }
+
+ /** Pages that would render larger than this many pixels are skipped; -1
for no limit. */
+ public void setMaxImagePixels(Long maxImagePixels) {
+ if (maxImagePixels != null && maxImagePixels < 1 && maxImagePixels !=
-1) {
+ throw new IllegalArgumentException(
+ "maxImagePixels must be -1 (no limit) or at least 1, got:
" + maxImagePixels);
+ }
+ this.maxImagePixels = maxImagePixels;
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/renderer/pdf/pdfbox/PDFBoxRenderer.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/renderer/pdf/pdfbox/PDFBoxRenderer.java
index d00c6e1297..49caee0a4a 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/renderer/pdf/pdfbox/PDFBoxRenderer.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/renderer/pdf/pdfbox/PDFBoxRenderer.java
@@ -27,6 +27,8 @@ import java.util.Set;
import org.apache.pdfbox.Loader;
import org.apache.pdfbox.io.RandomAccessRead;
import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.rendering.ImageType;
import org.apache.pdfbox.rendering.PDFRenderer;
import org.apache.pdfbox.tools.imageio.ImageIOUtil;
@@ -48,6 +50,7 @@ import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.pdf.PDFParser;
import org.apache.tika.parser.pdf.PDFParserConfig;
import org.apache.tika.parser.pdf.PDFRandomAccess;
+import org.apache.tika.parser.pdf.RenderingConfig;
import org.apache.tika.renderer.PageBasedRenderResults;
import org.apache.tika.renderer.PageRangeRequest;
import org.apache.tika.renderer.RenderRequest;
@@ -182,7 +185,8 @@ public class PDFBoxRenderer implements PDDocumentRenderer {
try {
m.set(TikaPagedText.PAGE_NUMBER, i);
m.set(TikaPagedText.PAGE_ROTATION,
(double)pdDocument.getPage(i - 1).getRotation());
- results.add(renderPage(renderer, id, i, m, parseContext));
+ results.add(renderPage(renderer, pdDocument.getPage(i - 1),
id, i, m,
+ parseContext));
} catch (IOException e) {
EmbeddedDocumentUtil.recordException(e, m, parseContext);
results.add(new RenderResult(RenderResult.STATUS.EXCEPTION,
id, null, m));
@@ -190,24 +194,33 @@ public class PDFBoxRenderer implements PDDocumentRenderer
{
}
}
- protected RenderResult renderPage(PDFRenderer renderer, int id, int
pageNumber,
+ protected RenderResult renderPage(PDFRenderer renderer, PDPage page, int
id, int pageNumber,
Metadata metadata, ParseContext
parseContext)
throws IOException {
-
+ int dpi = getDPI(parseContext);
+ long maxPixels = getMaxImagePixels(parseContext);
+ if (maxPixels > 0) {
+ PDRectangle mediaBox = page.getMediaBox();
+ long estWidth = (long) Math.ceil(mediaBox.getWidth() / 72.0 * dpi);
+ long estHeight = (long) Math.ceil(mediaBox.getHeight() / 72.0 *
dpi);
+ if (estWidth * estHeight > maxPixels) {
+ throw new IOException("page " + pageNumber + " would render to
" + estWidth + "x"
+ + estHeight + " pixels at " + dpi + " dpi, above
maxImagePixels "
+ + maxPixels);
+ }
+ }
Path tmpFile = Files.createTempFile("tika-pdfbox-rendering-",
"-" + id + "-" + pageNumber + "." +
getImageFormatName(parseContext));
try {
long start = System.currentTimeMillis();
//TODO: parameterize whether or not to un-rotate page?
BufferedImage image = renderer.renderImageWithDPI(
- pageNumber - 1,
- getDPI(parseContext),
- getImageType(parseContext));
+ pageNumber - 1, dpi, getImageType(parseContext));
long renderingElapsed = System.currentTimeMillis() - start;
metadata.set(PDFBOX_RENDERING_TIME_MS, renderingElapsed);
start = System.currentTimeMillis();
try (OutputStream os = Files.newOutputStream(tmpFile)) {
- ImageIOUtil.writeImage(image,
getImageFormatName(parseContext), os, getDPI(parseContext),
+ ImageIOUtil.writeImage(image,
getImageFormatName(parseContext), os, dpi,
getImageQuality(parseContext));
}
long elapsedWrite = System.currentTimeMillis() - start;
@@ -244,34 +257,41 @@ public class PDFBoxRenderer implements PDDocumentRenderer
{
}
protected float getImageQuality(ParseContext parseContext) {
- PDFParserConfig pdfParserConfig =
parseContext.get(PDFParserConfig.class);
- if (pdfParserConfig == null) {
- return defaultImageQuality;
- }
- return pdfParserConfig.getOcr().getImageQuality();
+ RenderingConfig settings = settings(parseContext);
+ return settings == null ? defaultImageQuality :
settings.getImageQuality();
}
protected int getDPI(ParseContext parseContext) {
- PDFParserConfig pdfParserConfig =
parseContext.get(PDFParserConfig.class);
- if (pdfParserConfig == null) {
- return defaultDPI;
- }
- return pdfParserConfig.getOcr().getDpi();
+ RenderingConfig settings = settings(parseContext);
+ return settings == null ? defaultDPI : settings.getDpi();
}
protected ImageType getImageType(ParseContext parseContext) {
- PDFParserConfig pdfParserConfig =
parseContext.get(PDFParserConfig.class);
- if (pdfParserConfig == null) {
- return defaultImageType;
- }
- return pdfParserConfig.getOcr().getImageType().getPdfBoxImageType();
+ RenderingConfig settings = settings(parseContext);
+ return settings == null ? defaultImageType :
settings.getImageType().getPdfBoxImageType();
}
protected String getImageFormatName(ParseContext parseContext) {
- PDFParserConfig pdfParserConfig =
parseContext.get(PDFParserConfig.class);
- if (pdfParserConfig == null) {
- return defaultImageFormatName;
+ RenderingConfig settings = settings(parseContext);
+ return settings == null ? defaultImageFormatName :
settings.getImageFormat().getFormatName();
+ }
+
+ /** -1 for no limit. */
+ protected long getMaxImagePixels(ParseContext parseContext) {
+ RenderingConfig settings = settings(parseContext);
+ return settings == null ? -1 : settings.getMaxImagePixels();
+ }
+
+ /**
+ * The page-image settings the PDF parser scopes around an emitted render,
else the OCR
+ * settings; null when no PDF parser config is in the context.
+ */
+ private static RenderingConfig settings(ParseContext parseContext) {
+ RenderingConfig emitted = parseContext.get(RenderingConfig.class);
+ if (emitted != null) {
+ return emitted;
}
- return pdfParserConfig.getOcr().getImageFormat().getFormatName();
+ PDFParserConfig pdfParserConfig =
parseContext.get(PDFParserConfig.class);
+ return pdfParserConfig == null ? null :
RenderingConfig.from(pdfParserConfig.getOcr());
}
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFRenderingConfigTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFRenderingConfigTest.java
new file mode 100644
index 0000000000..3f97462bce
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFRenderingConfigTest.java
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.pdf;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.awt.image.BufferedImage;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import javax.imageio.ImageIO;
+
+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.rendering.PDFRenderer;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.HttpHeaders;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.enricher.CompositeContentEnricher;
+import org.apache.tika.parser.enricher.TextRecognizer;
+import org.apache.tika.renderer.RenderResult;
+import org.apache.tika.renderer.pdf.pdfbox.PDFBoxRenderer;
+import org.apache.tika.sax.ToXMLContentHandler;
+import org.apache.tika.sax.XHTMLContentHandler;
+
+/**
+ * {@code "rendering"} sets the page images the parser emits; OCR keeps
rendering with the
+ * {@code ocr} settings, and the two share one render only when it would be
the same image.
+ */
+public class PDFRenderingConfigTest extends TikaTest {
+
+ private static final String TWO_PAGES = "testPDF_bookmarks.pdf";
+
+ @ParameterizedTest
+ @EnumSource(value = PDFParserConfig.IMAGE_STRATEGY.class,
+ names = {"RENDER_PAGES_BEFORE_PARSE", "RENDER_PAGES_AT_PAGE_END"})
+ public void testPageImageUsesRenderingBlock(PDFParserConfig.IMAGE_STRATEGY
strategy)
+ throws Exception {
+ PDFParserConfig config = new PDFParserConfig();
+ config.setImageStrategy(strategy);
+ config.getRendering().setDpi(36);
+ config.getRendering().setImageType(OcrConfig.ImageType.RGB);
+ ImageSink pages = new ImageSink();
+
+ parse(config, pages, null);
+
+ assertEquals(2, pages.images.size());
+ for (Shape page : pages.images) {
+ assertEquals(pixelsAcross(36), page.width, "36 dpi, not ocr's
300");
+ assertEquals(3, page.colorComponents, "RGB, not ocr's GRAY");
+ }
+ }
+
+ @ParameterizedTest
+ @EnumSource(value = PDFParserConfig.IMAGE_STRATEGY.class,
+ names = {"RENDER_PAGES_BEFORE_PARSE", "RENDER_PAGES_AT_PAGE_END"})
+ public void testNoBlockFollowsOcr(PDFParserConfig.IMAGE_STRATEGY strategy)
throws Exception {
+ PDFParserConfig config = new PDFParserConfig();
+ config.setImageStrategy(strategy);
+ config.getOcr().setDpi(36);
+ ImageSink pages = new ImageSink();
+
+ parse(config, pages, null);
+
+ assertEquals(2, pages.images.size());
+ for (Shape page : pages.images) {
+ assertEquals(pixelsAcross(36), page.width, "the 4.0 behaviour:
ocr's dpi");
+ assertEquals(1, page.colorComponents, "the 4.0 behaviour: ocr's
GRAY");
+ }
+ }
+
+ @ParameterizedTest
+ @EnumSource(value = PDFParserConfig.IMAGE_STRATEGY.class,
+ names = {"RENDER_PAGES_BEFORE_PARSE", "RENDER_PAGES_AT_PAGE_END"})
+ public void testOcrKeepsItsOwnSettings(PDFParserConfig.IMAGE_STRATEGY
strategy)
+ throws Exception {
+ PDFParserConfig config = new PDFParserConfig();
+ config.setImageStrategy(strategy);
+ // OCR-only text goes through OCR2XHTML, which emits no page images at
page end
+ config.setText(PDFParserConfig.TextPolicy.EXTRACT_AND_OCR);
+ config.getOcr().setDpi(72);
+ config.getRendering().setDpi(36);
+ config.getRendering().setImageType(OcrConfig.ImageType.RGB);
+ ImageSink pages = new ImageSink();
+ ImageSink ocr = new ImageSink();
+
+ parse(config, pages, ocr);
+
+ assertEquals(2, ocr.images.size(), "the recognizer saw each page");
+ for (Shape page : ocr.images) {
+ assertEquals(pixelsAcross(72), page.width, "OCR renders at
ocr.dpi");
+ assertEquals(1, page.colorComponents, "OCR renders in
ocr.imageType");
+ }
+ assertEquals(2, pages.images.size(), "each page was emitted");
+ for (Shape page : pages.images) {
+ assertEquals(pixelsAcross(36), page.width);
+ assertEquals(3, page.colorComponents);
+ }
+ }
+
+ @Test
+ public void testBeforeParseRenderServesOcrOnlyWhenItIsTheSameImage()
throws Exception {
+ PDFParserConfig same = new PDFParserConfig();
+
same.setImageStrategy(PDFParserConfig.IMAGE_STRATEGY.RENDER_PAGES_BEFORE_PARSE);
+ same.setText(PDFParserConfig.TextPolicy.EXTRACT_AND_OCR);
+ same.getOcr().setDpi(36);
+ assertEquals(2, renders(same), "one render per page feeds both the
output and OCR");
+
+ PDFParserConfig different = new PDFParserConfig();
+
different.setImageStrategy(PDFParserConfig.IMAGE_STRATEGY.RENDER_PAGES_BEFORE_PARSE);
+ different.setText(PDFParserConfig.TextPolicy.EXTRACT_AND_OCR);
+ different.getOcr().setDpi(36);
+ different.getRendering().setImageType(OcrConfig.ImageType.RGB);
+ assertEquals(4, renders(different), "OCR renders again rather than
read the RGB page");
+ }
+
+ @ParameterizedTest
+ @EnumSource(value = PDFParserConfig.IMAGE_STRATEGY.class,
+ names = {"RENDER_PAGES_BEFORE_PARSE", "RENDER_PAGES_AT_PAGE_END"})
+ public void testMaxImagePixelsSkipsThePageWithAWarning(
+ PDFParserConfig.IMAGE_STRATEGY strategy) throws Exception {
+ PDFParserConfig config = new PDFParserConfig();
+ config.setImageStrategy(strategy);
+ config.getRendering().setDpi(36);
+ config.getRendering().setMaxImagePixels(1000L);
+ ImageSink pages = new ImageSink();
+
+ Metadata metadata = parse(config, pages, null);
+
+ assertEquals(0, pages.images.size(), "no page fits in 1000 pixels");
+ String[] warnings =
metadata.getValues(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING);
+ assertEquals(2, warnings.length, "one warning per skipped page");
+ assertTrue(warnings[0].contains("maxImagePixels"), warnings[0]);
+ }
+
+ @Test
+ public void testJsonConfig() throws Exception {
+ ParseContext context = new ParseContext();
+ context.setJsonConfig("pdf-parser", "{\"imageStrategy\":
\"RENDER_PAGES_AT_PAGE_END\","
+ + " \"rendering\": {\"dpi\": 36, \"imageType\": \"RGB\",
\"imageFormat\": \"JPEG\","
+ + " \"imageQuality\": 0.9, \"maxImagePixels\": -1}}");
+ List<Metadata> metadataList = getRecursiveMetadata(TWO_PAGES, context);
+ List<Metadata> renderings = new ArrayList<>();
+ for (Metadata m : metadataList) {
+ if (TikaCoreProperties.EmbeddedResourceType.RENDERING.name()
+ .equals(m.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE)))
{
+ renderings.add(m);
+ }
+ }
+ assertEquals(2, renderings.size());
+ assertEquals("image/jpeg",
renderings.get(0).get(HttpHeaders.CONTENT_TYPE));
+ }
+
+ @Test
+ public void testValidation() {
+ RenderingConfig config = new RenderingConfig();
+ assertThrows(IllegalArgumentException.class, () -> config.setDpi(0));
+ assertThrows(IllegalArgumentException.class, () ->
config.setImageQuality(1.5f));
+ assertThrows(IllegalArgumentException.class, () ->
config.setMaxImagePixels(0L));
+ config.setDpi(null);
+ config.setMaxImagePixels(-1L);
+ }
+
+ @Test
+ public void testResolveFallsBackPerField() {
+ OcrConfig ocr = new OcrConfig();
+ ocr.setDpi(150);
+ RenderingConfig block = new RenderingConfig();
+ block.setImageType(OcrConfig.ImageType.RGB);
+ RenderingConfig resolved = block.resolve(ocr);
+ assertEquals(150, resolved.getDpi());
+ assertEquals(OcrConfig.ImageType.RGB, resolved.getImageType());
+ assertEquals(ocr.getImageFormat(), resolved.getImageFormat());
+ assertTrue(new
RenderingConfig().resolve(ocr).rendersSameImageAs(RenderingConfig.from(ocr)));
+ assertTrue(!resolved.rendersSameImageAs(RenderingConfig.from(ocr)));
+ }
+
+ /** Parses with a PDFParser whose embedded image parser and (optional)
recognizer record what they see. */
+ private Metadata parse(PDFParserConfig config, ImageSink pages, ImageSink
ocr)
+ throws Exception {
+ PDFParser parser = new PDFParser();
+ if (ocr != null) {
+ parser.setContentEnrichers(new
CompositeContentEnricher(List.of(ocr)));
+ }
+ ParseContext context = new ParseContext();
+ context.set(PDFParserConfig.class, config);
+ context.set(Parser.class, new AutoDetectParser(pages));
+ Metadata metadata = new Metadata();
+ try (TikaInputStream tis = TikaInputStream.get(
+ getResourceAsStream("/test-documents/" + TWO_PAGES))) {
+ parser.parse(tis, new ToXMLContentHandler(), metadata, context);
+ }
+ return metadata;
+ }
+
+ private int renders(PDFParserConfig config) throws Exception {
+ CountingRenderer renderer = new CountingRenderer();
+ PDFParser parser = new PDFParser();
+ parser.setRenderer(renderer);
+ parser.setContentEnrichers(new CompositeContentEnricher(List.of(new
ImageSink())));
+ ParseContext context = new ParseContext();
+ context.set(PDFParserConfig.class, config);
+ context.set(Parser.class, new AutoDetectParser(new ImageSink()));
+ try (TikaInputStream tis = TikaInputStream.get(
+ getResourceAsStream("/test-documents/" + TWO_PAGES))) {
+ parser.parse(tis, new ToXMLContentHandler(), new Metadata(),
context);
+ }
+ return renderer.pages;
+ }
+
+ private int pixelsAcross(int dpi) throws IOException {
+ try (InputStream is = getResourceAsStream("/test-documents/" +
TWO_PAGES);
+ PDDocument document = Loader.loadPDF(is.readAllBytes())) {
+ // PDFBox floors the scaled page width
+ PDPage page = document.getPage(0);
+ return (int) Math.max(1, Math.floor(page.getMediaBox().getWidth()
* dpi / 72f));
+ }
+ }
+
+ private static final class Shape {
+ final int width;
+ final int colorComponents;
+
+ Shape(BufferedImage image) {
+ this.width = image.getWidth();
+ this.colorComponents =
image.getColorModel().getNumColorComponents();
+ }
+ }
+
+ /** Reads every image it is handed, as an embedded-document parser or as
the OCR engine. */
+ private static final class ImageSink implements Parser, TextRecognizer {
+ private static final long serialVersionUID = 1L;
+ final List<Shape> images = new ArrayList<>();
+
+ @Override
+ public boolean recognizesText(ParseContext context) {
+ return true;
+ }
+
+ @Override
+ public Set<MediaType> getSupportedTypes(ParseContext context) {
+ return Set.of(MediaType.image("png"), MediaType.image("jpeg"));
+ }
+
+ @Override
+ public void parse(TikaInputStream tis, ContentHandler handler,
Metadata metadata,
+ ParseContext context) throws IOException,
SAXException, TikaException {
+ BufferedImage image = ImageIO.read(tis);
+ assertNotNull(image, "not an image: " +
metadata.get(HttpHeaders.CONTENT_TYPE));
+ images.add(new Shape(image));
+ XHTMLContentHandler xhtml = new XHTMLContentHandler(handler,
metadata);
+ xhtml.startDocument();
+ xhtml.characters("seen");
+ xhtml.endDocument();
+ }
+ }
+
+ private static final class CountingRenderer extends PDFBoxRenderer {
+ private static final long serialVersionUID = 1L;
+ int pages;
+
+ @Override
+ protected RenderResult renderPage(PDFRenderer renderer, PDPage page,
int id,
+ int pageNumber, Metadata metadata,
+ ParseContext parseContext) throws
IOException {
+ pages++;
+ return super.renderPage(renderer, page, id, pageNumber, metadata,
parseContext);
+ }
+ }
+}
