This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 5da77f7e29 TIKA-4889: pdf-parser.text says where a page's text comes
from (#3170)
5da77f7e29 is described below
commit 5da77f7e29cbc04463f931939167da572b2a024b
Author: Tim Allison <[email protected]>
AuthorDate: Mon Sep 14 20:54:02 2026 -0400
TIKA-4889: pdf-parser.text says where a page's text comes from (#3170)
---
CHANGES.txt | 7 ++
.../advanced/integration-testing/tika-app.adoc | 4 +-
.../advanced/integration-testing/tika-server.adoc | 4 +-
.../pages/configuration/inference-recipes.adoc | 6 +-
.../ROOT/pages/configuration/inference.adoc | 4 +-
.../pages/configuration/parsers/pdf-parser.adoc | 48 ++++++++++----
.../ROOT/pages/developers/serialization.adoc | 8 +--
.../migration-to-4x/migrating-tika-server-4x.adoc | 2 +-
.../pages/migration-to-4x/migrating-to-4x.adoc | 13 +++-
.../ROOT/pages/using-tika/server/index.adoc | 4 +-
.../apache/tika/cli/XmlToJsonConfigConverter.java | 10 +++
.../tika/cli/XmlToJsonConfigConverterTest.java | 5 +-
.../apache/tika/parser/crypto/TSDParserTest.java | 3 +-
.../org/apache/tika/parser/pdf/PDFParserTest.java | 65 +++++++++++++------
.../resources/config-examples/pdf-parser-full.json | 13 ++--
.../apache/tika/parser/pdf/AbstractPDF2XHTML.java | 36 +++++------
.../java/org/apache/tika/parser/pdf/OCR2XHTML.java | 5 +-
.../java/org/apache/tika/parser/pdf/OcrConfig.java | 41 +++++++++---
.../java/org/apache/tika/parser/pdf/PDFParser.java | 7 +-
.../apache/tika/parser/pdf/PDFParserConfig.java | 36 ++++++++++-
.../tika/parser/pdf/PDFMarkedContentGateTest.java | 2 +-
.../org/apache/tika/parser/pdf/PDFParserTest.java | 74 +++++++++++++++++-----
.../apache/tika/config/loader/PresetRegistry.java | 2 +-
.../org/apache/tika/config/loader/TikaLoader.java | 4 +-
.../customocr/tika-config-rendered.json | 2 +-
.../config-examples/server-with-parsers.json | 4 +-
.../server/standard/InferenceBindingsWireTest.java | 2 +-
27 files changed, 294 insertions(+), 117 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 90d23060af..5f0746ff9c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,12 @@
Release 4.1.0 - unreleased
+ * PDF: "pdf-parser": {"text": ...} says where a page's text comes from:
+ EXTRACT, AUTO (default), EXTRACT_AND_OCR, OCR, or the new NONE, which
+ writes no text from any source while pages are still rendered for
+ annotators and PAGES inference bindings. The 4.0 "ocr": {"strategy"}
+ spellings load as aliases and dump as "text"; OcrConfig.getStrategy()
+ is removed, read PDFParserConfig.getText() (TIKA-4889).
+
* Engines are closed with the config that loaded them: Engine is
Closeable (no-op by default), EngineRegistry closes every engine once,
and the pipes server closes the registry at shutdown, so an
diff --git a/docs/modules/ROOT/pages/advanced/integration-testing/tika-app.adoc
b/docs/modules/ROOT/pages/advanced/integration-testing/tika-app.adoc
index 5cfad06af8..5f325b4fd3 100644
--- a/docs/modules/ROOT/pages/advanced/integration-testing/tika-app.adoc
+++ b/docs/modules/ROOT/pages/advanced/integration-testing/tika-app.adoc
@@ -259,9 +259,7 @@ Create `/tmp/tika-app-test/my-config.json`:
"pdf-parser": {
"extractActions": true,
"extractInlineImages": true,
- "ocr": {
- "strategy": "NO_OCR"
- }
+ "text": "EXTRACT"
}
},
{
diff --git
a/docs/modules/ROOT/pages/advanced/integration-testing/tika-server.adoc
b/docs/modules/ROOT/pages/advanced/integration-testing/tika-server.adoc
index dca58ed791..793e2e59de 100644
--- a/docs/modules/ROOT/pages/advanced/integration-testing/tika-server.adoc
+++ b/docs/modules/ROOT/pages/advanced/integration-testing/tika-server.adoc
@@ -283,7 +283,7 @@ curl -s -X POST -F "[email protected]" -H "Accept:
application/json" http://loca
[source,bash]
----
curl -s -X POST -F "[email protected]" \
- -F 'config={"pdf-parser":{"ocr":{"strategy":"NO_OCR"}}}' \
+ -F 'config={"pdf-parser":{"text":"EXTRACT"}}' \
-H "Accept: application/json" \
http://localhost:9998/meta/config
----
@@ -302,7 +302,7 @@ request time). Supplying the nested form currently returns
HTTP 500 (the blocked
[source,bash]
----
curl -s -X POST -F "file=@test_recursive_embedded.docx" \
- -F 'config={"pdf-parser":{"ocr":{"strategy":"NO_OCR"}}}' \
+ -F 'config={"pdf-parser":{"text":"EXTRACT"}}' \
http://localhost:9998/unpack/all/config -o /tmp/unpack-config.zip
unzip -l /tmp/unpack-config.zip
----
diff --git a/docs/modules/ROOT/pages/configuration/inference-recipes.adoc
b/docs/modules/ROOT/pages/configuration/inference-recipes.adoc
index 2b6b15d5d5..777326a4a1 100644
--- a/docs/modules/ROOT/pages/configuration/inference-recipes.adoc
+++ b/docs/modules/ROOT/pages/configuration/inference-recipes.adoc
@@ -72,7 +72,7 @@ settings live under `ocr` and are sized here for an image
model rather than Tess
"parsers": [
{ "default-parser": {} },
{ "pdf-parser": { "inference": { "input": ["PAGES"] },
- "ocr": { "strategy": "NO_OCR", "dpi": 100, "imageType":
"RGB" } } }
+ "text": "EXTRACT", "ocr": { "dpi": 100, "imageType":
"RGB" } } }
]
}
----
@@ -120,7 +120,7 @@ every other document is embedded by its text. List
`["TEXT", "PAGES"]` on the PD
"parsers": [
{ "default-parser": {} },
{ "pdf-parser": { "inference": { "input": ["PAGES"] },
- "ocr": { "strategy": "NO_OCR", "dpi": 100, "imageType":
"RGB" } } }
+ "text": "EXTRACT", "ocr": { "dpi": 100, "imageType":
"RGB" } } }
]
}
----
@@ -136,7 +136,7 @@ The recognizer stays on for embedded and standalone images;
the PDF parser never
"text-recognizers": [ { "engine": "tesseract" } ],
"parsers": [
{ "default-parser": {} },
- { "pdf-parser": { "ocr": { "strategy": "NO_OCR" } } }
+ { "pdf-parser": { "text": "EXTRACT" } }
]
}
----
diff --git a/docs/modules/ROOT/pages/configuration/inference.adoc
b/docs/modules/ROOT/pages/configuration/inference.adoc
index cb2f65cdc1..4eb115027d 100644
--- a/docs/modules/ROOT/pages/configuration/inference.adoc
+++ b/docs/modules/ROOT/pages/configuration/inference.adoc
@@ -127,7 +127,7 @@ only when its own config releases pages:
"parsers": [
{ "default-parser": {} },
{ "pdf-parser": { "inference": { "input": ["PAGES"] },
- "ocr": { "strategy": "NO_OCR", "dpi": 100, "imageType":
"RGB" } } }
+ "text": "EXTRACT", "ocr": { "dpi": 100, "imageType":
"RGB" } } }
]
}
----
@@ -135,7 +135,7 @@ only when its own config releases pages:
`pdf-parser.inference.input` lists what the PDF releases to inference: `TEXT`,
the default
(text bindings follow), and `PAGES`. With `PAGES`, and a `PAGES` binding that
runs for the
request, the PDF renders every page once and hands the same image to the OCR
engine, when
-`ocr.strategy` asks for it, and to the bindings: a scanned page costs one
render whether one
+`text` asks for it, and to the bindings: a scanned page costs one render
whether one
consumer or two see it. Each page is one unit. Its vector lands on the PDF's
own `tk:chunks`
with a `paginated` locator naming the page, and `maxChunks` is the number of
pages one document
sends. Nothing is rendered for inference unless a `PAGES` binding runs, so the
block can stay on
diff --git a/docs/modules/ROOT/pages/configuration/parsers/pdf-parser.adoc
b/docs/modules/ROOT/pages/configuration/parsers/pdf-parser.adoc
index 2415d9f6e8..b42dd60ad5 100644
--- a/docs/modules/ROOT/pages/configuration/parsers/pdf-parser.adoc
+++ b/docs/modules/ROOT/pages/configuration/parsers/pdf-parser.adoc
@@ -39,6 +39,33 @@ include::example$pdf-parser-full.json[]
----
icon:github[]
https://github.com/apache/tika/blob/main/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/resources/config-examples/pdf-parser-full.json[View
source on GitHub]
+== Where text comes from (`text`)
+
+`text` says where a page's text comes from; it is the parser's own setting,
beside the `ocr`
+block, and can be set per request:
+
+`EXTRACT`::
+The content stream only; pages are never rendered for OCR.
+`AUTO` (default)::
+The content stream, and OCR of a page whose extracted text is poor by the
`ocr.strategyAuto`
+thresholds; the page's extracted text is then *replaced* by the engine's
output when the engine is
+a text recognizer (Tesseract, Tess4J, a VLM with `textRecognizer` true) that
actually wrote text,
+and kept unchanged otherwise, including when no engine is available,
`maxPagesToOcr` is spent, or
+the engine times out. `AUTO` with no engine behaves as `EXTRACT`.
+`EXTRACT_AND_OCR`::
+Both, on every page.
+`OCR`::
+The engine's output only; the content stream is not read. Fails the parse when
no engine covers
+the render type, unless the request switched the recognizers off.
+`NONE`::
+No text from any source: no content stream, no OCR, no bookmark, form or
annotation text.
+Metadata, attachments and page renders still happen, so a `PAGES` inference
binding or a page
+annotator runs at no text cost.
+
+The 4.0 spelling `"ocr": {"strategy": ...}` still loads as an alias: `NO_OCR`
is `EXTRACT`,
+`OCR_ONLY` is `OCR`, `OCR_AND_TEXT_EXTRACTION` is `EXTRACT_AND_OCR`. When both
are set, `text`
+wins. A config dump writes `text`.
+
== OCR of pages (`ocr`)
The PDF parser does not OCR anything itself. It renders pages and hands each
rendered image to the
@@ -46,16 +73,10 @@ text recognizer resolved for the render type (`image/png`
by default): the engin
top-level `text-recognizers` list, or, with no list, the one found among the
loaded parsers and
logged at startup. See
xref:configuration/index.adoc#text-recognizers[Configuration]
for choosing the engine and
xref:configuration/parsers/tesseract-ocr-parser.adoc[Tesseract OCR]
-for its options. The `ocr` block controls only rendering and when OCR runs:
+for its options. The `ocr` block controls only rendering and the `AUTO`
verdict:
-`strategy`::
-`NO_OCR` never renders. `OCR_ONLY` emits the engine's output instead of
extracted text.
-`OCR_AND_TEXT_EXTRACTION` emits both. `AUTO`, the default, renders a page only
when its extracted
-text is poor by the `strategyAuto` thresholds; the page's extracted text is
then *replaced* by the
-engine's output when the engine is a text recognizer (Tesseract, Tess4J, a VLM
with
-`textRecognizer` true) that actually wrote text, and kept unchanged otherwise,
including when no
-engine is available, `maxPagesToOcr` is spent, or the engine times out. `AUTO`
with no engine
-behaves as `NO_OCR`.
+`strategyAuto`::
+Per-page character thresholds below which `text: AUTO` sends the page to OCR.
`dpi`, `imageType`, `imageFormat`, `imageQuality`, `renderingStrategy`::
How pages are rendered before they reach the engine.
`maxPagesToOcr`, `maxImagePixels`::
@@ -67,10 +88,10 @@ page that is one full-page image is therefore OCR'd twice
when both are on.
Page renders emitted as embedded documents (`imageStrategy` set to
`RENDER_PAGES_AT_PAGE_END` or
`RENDER_PAGES_BEFORE_PARSE`) are enriched by the PDF parser itself, once per
page: the text
-recognizer where `strategy` says so, and every other enricher (an image
embedder, a tagging VLM)
+recognizer where `text` says so, and every other enricher (an image embedder,
a tagging VLM)
on every page, with its results such as `tk:chunks` landing on the PDF. The
embedded copy of a
render is parsed for its metadata but not enriched again, so no page reaches
the same engine
-twice, and `NO_OCR` means no page OCR anywhere while the other enrichers still
see every page.
+twice, and `EXTRACT` means no page OCR anywhere while the other enrichers
still see every page.
`pdf:ocr-page-count` records how many pages tripped the verdict, whether or
not an engine ran.
@@ -137,5 +158,6 @@ What the parser releases to the
xref:configuration/inference.adoc#pages[inferenc
the `ocr` render, made once per page for OCR and inference alike, and nothing
is rendered
unless a `PAGES` binding runs for the request. Per request under `pdf-parser`
in
`parse-context`, like the `ocr` block.
-Every `ocr` option can be overridden per request through `parse-context` under
`pdf-parser`; the
-xref:using-tika/server/index.adoc[server documentation] shows the `NO_OCR`
override as a preset.
+`text` and every `ocr` option can be overridden per request through
`parse-context` under
+`pdf-parser`; the xref:using-tika/server/index.adoc[server documentation]
shows the `EXTRACT`
+override as a preset.
diff --git a/docs/modules/ROOT/pages/developers/serialization.adoc
b/docs/modules/ROOT/pages/developers/serialization.adoc
index 93e9f83ade..b430c1583f 100644
--- a/docs/modules/ROOT/pages/developers/serialization.adoc
+++ b/docs/modules/ROOT/pages/developers/serialization.adoc
@@ -56,7 +56,7 @@ Inside an array-valued section (`parsers`, `detectors`,
`encoding-detectors`,
`metadata-filters`) an element can be specified as:
* **String**: `"pdf-parser"` - creates instance with defaults
-* **Object**: `{"pdf-parser": {"ocr": {"strategy": "AUTO"}}}` - creates
configured instance
+* **Object**: `{"pdf-parser": {"text": "AUTO"}}` - creates configured instance
Object-valued sections such as `parse-context` key each entry by name instead,
with
the config object as the value.
@@ -220,9 +220,7 @@ Benefits:
{
"parse-context": {
"pdf-parser": {
- "ocr": {
- "strategy": "AUTO"
- },
+ "text": "AUTO",
"extractInlineImages": true
},
"commons-digester-factory": {
@@ -258,7 +256,7 @@ extends `SelfConfiguring` — are skipped by the wire-block
scan
(`ParseContextDeserializer.assertNoBlockedComponents`): their config subtree is
passed through to the component unexamined. So while a request cannot bind a
new
`Parser` from the wire, a request carrying
-`{"parse-context": {"pdf-parser": {"ocr": {"strategy":
"OCR_AND_TEXT_EXTRACTION"}}}}`
+`{"parse-context": {"pdf-parser": {"text": "EXTRACT_AND_OCR"}}}`
will reach `PDFParser` and take effect.
That is why per-request configuration is gated separately by
diff --git
a/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc
b/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc
index ac4afd013f..a0d86a47ba 100644
--- a/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc
+++ b/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc
@@ -67,7 +67,7 @@ POST endpoints accept multipart requests with a `file` part
and optional `config
# Parse with custom PDF parser settings (requires allowPerRequestConfig)
curl -X POST http://localhost:9998/tika/config/json \
-F "[email protected]" \
- -F
"config={\"pdf-parser\":{\"ocr\":{\"strategy\":\"NO_OCR\"}}};type=application/json"
+ -F "config={\"pdf-parser\":{\"text\":\"EXTRACT\"}};type=application/json"
----
== Breaking Changes
diff --git a/docs/modules/ROOT/pages/migration-to-4x/migrating-to-4x.adoc
b/docs/modules/ROOT/pages/migration-to-4x/migrating-to-4x.adoc
index 411dd85361..6799a2579b 100644
--- a/docs/modules/ROOT/pages/migration-to-4x/migrating-to-4x.adoc
+++ b/docs/modules/ROOT/pages/migration-to-4x/migrating-to-4x.adoc
@@ -241,7 +241,7 @@ What changes for you:
(`{"tesseract-ocr-parser": {"_mime-exclude": ["image/tiff"]}}`); on the
`default-parser` entry a
real-type filter also removes the parser for that type.
* An engine the classpath supplied is never dispatched to; `image-parser`
always invokes it, and
- the PDF parser's `ocr.strategy` controls pages. A
`tk:content-type-parser-override` of
+ the PDF parser's `text` policy controls pages. A
`tk:content-type-parser-override` of
`image/ocr-*` matches no parser any more and the document falls to the empty
fallback parser:
drop the override.
* `Content-Type` never carries an `ocr-` prefix any more, so extracted image
files get their real
@@ -256,6 +256,17 @@ What changes for you:
* VLM parsers gain `textRecognizer` (default `true`); set it to `false` for a
captioning or tagging
prompt so the PDF parser's `AUTO` OCR never replaces extracted text with the
VLM's output.
+=== PDF text policy: `ocr.strategy` becomes `text` (4.1.0+)
+
+The PDF parser's `"ocr": {"strategy": ...}` is now the parser's own `"text"`,
since it decides
+where every page's text comes from, not only whether OCR runs: `NO_OCR` is
`EXTRACT`, `OCR_ONLY`
+is `OCR`, `OCR_AND_TEXT_EXTRACTION` is `EXTRACT_AND_OCR`, `AUTO` is unchanged,
and `NONE` is new
+(no text from any source; pages are still rendered for annotators and
inference). The old
+spelling loads as an alias, `text` wins when both are set, and a config dump
writes `text`.
+The XML converter writes `text`. In Java, `OcrConfig.getStrategy()` is gone:
read
+`PDFParserConfig.getText()`; `OcrConfig.setStrategy` is deprecated. See
+xref:configuration/parsers/pdf-parser.adoc[PDF parser].
+
For the general serialization model and how JSON configuration works, see
xref:developers/serialization.adoc[Serialization and Configuration].
diff --git a/docs/modules/ROOT/pages/using-tika/server/index.adoc
b/docs/modules/ROOT/pages/using-tika/server/index.adoc
index 5c914e6692..fd3d81dfac 100644
--- a/docs/modules/ROOT/pages/using-tika/server/index.adoc
+++ b/docs/modules/ROOT/pages/using-tika/server/index.adoc
@@ -132,7 +132,7 @@ the catalog fails startup:
{
"presets": {
"some-catalog-preset": true,
- "no-ocr": { "pdf-parser": { "ocr": { "strategy": "NO_OCR" } } },
+ "no-ocr": { "pdf-parser": { "text": "EXTRACT" } },
"no-enrichment": { "text-recognizers": { "enabled": false }, "inference":
{ "enabled": false } }
}
}
@@ -303,7 +303,7 @@ POST with multipart for custom per-request configuration:
----
curl -X POST http://localhost:9998/tika/config/json \
-F "[email protected]" \
- -F
"config={\"pdf-parser\":{\"ocr\":{\"strategy\":\"NO_OCR\"}}};type=application/json"
+ -F "config={\"pdf-parser\":{\"text\":\"EXTRACT\"}};type=application/json"
----
Valid handler paths under `/tika/`: `text`, `html`, `xml`, `md`, `json` (bare
`/tika` returns
diff --git
a/tika-app/src/main/java/org/apache/tika/cli/XmlToJsonConfigConverter.java
b/tika-app/src/main/java/org/apache/tika/cli/XmlToJsonConfigConverter.java
index b7dab8a19b..e46a80a495 100644
--- a/tika-app/src/main/java/org/apache/tika/cli/XmlToJsonConfigConverter.java
+++ b/tika-app/src/main/java/org/apache/tika/cli/XmlToJsonConfigConverter.java
@@ -424,11 +424,21 @@ public class XmlToJsonConfigConverter {
it.remove();
}
}
+ // 4.1 spells the strategy as the parser's "text"; the ocr.strategy
alias still loads
+ Object strategy = ocr.remove("strategy");
+ if (strategy != null && !config.containsKey("text")) {
+ config.put("text",
OCR_STRATEGY_TO_TEXT.getOrDefault(String.valueOf(strategy),
+ String.valueOf(strategy)));
+ }
if (!ocr.isEmpty()) {
config.put("ocr", ocr);
}
}
+ private static final Map<String, String> OCR_STRATEGY_TO_TEXT = Map.of(
+ "NO_OCR", "EXTRACT", "AUTO", "AUTO", "OCR_ONLY", "OCR",
+ "OCR_AND_TEXT_EXTRACTION", "EXTRACT_AND_OCR");
+
/**
* Converts a <params> element to a map of parameter names to values.
*/
diff --git
a/tika-app/src/test/java/org/apache/tika/cli/XmlToJsonConfigConverterTest.java
b/tika-app/src/test/java/org/apache/tika/cli/XmlToJsonConfigConverterTest.java
index d6ad5dcf84..447f6e177c 100644
---
a/tika-app/src/test/java/org/apache/tika/cli/XmlToJsonConfigConverterTest.java
+++
b/tika-app/src/test/java/org/apache/tika/cli/XmlToJsonConfigConverterTest.java
@@ -392,10 +392,11 @@ public class XmlToJsonConfigConverterTest {
assertFalse(pdf.has("ocrStrategy"), "flat ocrStrategy should be
nested, not left at top level");
assertFalse(pdf.has("ocrDPI"), "flat ocrDPI should be nested, not left
at top level");
- // ...and merged into the single nested "ocr" object.
+ // ...and merged into the single nested "ocr" object, the strategy as
"text".
JsonNode ocr = pdf.get("ocr");
assertNotNull(ocr, "merged nested ocr object should be present");
- assertEquals("NO_OCR", ocr.get("strategy").asText(),
+ assertFalse(ocr.has("strategy"), "the strategy is written as the 4.1
\"text\" key");
+ assertEquals("EXTRACT", pdf.get("text").asText(),
"explicit nested strategy must win over the legacy flat
ocrStrategy");
assertEquals(200, ocr.get("dpi").asInt(),
"legacy flat ocrDPI must be migrated into ocr.dpi");
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/crypto/TSDParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/crypto/TSDParserTest.java
index ce16c22781..01d502b527 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/crypto/TSDParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/crypto/TSDParserTest.java
@@ -28,7 +28,6 @@ import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.pdf.OcrConfig;
import org.apache.tika.parser.pdf.PDFParserConfig;
public class TSDParserTest extends TikaTest {
@@ -37,7 +36,7 @@ public class TSDParserTest extends TikaTest {
public void testBrokenPdf() throws Exception {
ParseContext parseContext = new ParseContext();
PDFParserConfig config = new PDFParserConfig();
- config.getOcr().setStrategy(OcrConfig.Strategy.NO_OCR);
+ config.setText(PDFParserConfig.TextPolicy.EXTRACT);
parseContext.set(PDFParserConfig.class, config);
//make sure that embedded file appears in list
//and make sure embedded exception is recorded
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
index 52ac5255bb..20a04c8d2d 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
@@ -18,6 +18,7 @@ package org.apache.tika.parser.pdf;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -37,6 +38,8 @@ import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Disabled;
@@ -46,6 +49,7 @@ import org.xml.sax.SAXException;
import org.apache.tika.TikaLoaderHelper;
import org.apache.tika.TikaTest;
+import org.apache.tika.config.loader.TikaLoader;
import org.apache.tika.config.loader.TikaObjectMapperFactory;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.extractor.ContainerExtractor;
@@ -115,7 +119,7 @@ public class PDFParserTest extends TikaTest {
private static ParseContext NO_OCR() {
PDFParserConfig config = new PDFParserConfig();
- config.getOcr().setStrategy(OcrConfig.Strategy.NO_OCR);
+ config.setText(PDFParserConfig.TextPolicy.EXTRACT);
ParseContext context = new ParseContext();
context.set(PDFParserConfig.class, config);
return context;
@@ -232,7 +236,7 @@ public class PDFParserTest extends TikaTest {
PDFParserConfig config = new PDFParserConfig();
config.setExtractInlineImages(true);
config.setExtractUniqueInlineImagesOnly(false);
- config.getOcr().setStrategy(OcrConfig.Strategy.NO_OCR);
+ config.setText(PDFParserConfig.TextPolicy.EXTRACT);
context.set(org.apache.tika.parser.pdf.PDFParserConfig.class, config);
context.set(org.apache.tika.parser.Parser.class, p);
@@ -262,20 +266,22 @@ public class PDFParserTest extends TikaTest {
public void testEmbeddedDocsWithOCROnly() throws Exception {
assumeTrue(canRunOCR(), "can't run OCR");
//test default is "auto"
- assertEquals(OcrConfig.Strategy.AUTO, new
PDFParserConfig().getOcr().getStrategy());
+ assertEquals(PDFParserConfig.TextPolicy.AUTO, new
PDFParserConfig().getText());
testStrategy(null);
//now test other options
- for (OcrConfig.Strategy strategy : OcrConfig.Strategy.values()) {
- testStrategy(strategy);
+ for (PDFParserConfig.TextPolicy strategy :
PDFParserConfig.TextPolicy.values()) {
+ if (strategy != PDFParserConfig.TextPolicy.NONE) {
+ testStrategy(strategy);
+ }
}
}
- private void testStrategy(OcrConfig.Strategy strategy) throws Exception {
+ private void testStrategy(PDFParserConfig.TextPolicy strategy) throws
Exception {
//make sure everything works with regular xml _and_ with recursive
ParseContext context = new ParseContext();
if (strategy != null) {
PDFParserConfig config = new PDFParserConfig();
- config.getOcr().setStrategy(strategy);
+ config.setText(strategy);
context.set(PDFParserConfig.class, config);
};
PDFParserConfig config = context.get(PDFParserConfig.class, new
PDFParserConfig());
@@ -291,7 +297,7 @@ public class PDFParserTest extends TikaTest {
}
assertContains("Haystack", xmlResult.xml);
assertContains("Needle", xmlResult.xml);
- if (strategy == null || strategy != OcrConfig.Strategy.NO_OCR) {
+ if (strategy == null || strategy !=
PDFParserConfig.TextPolicy.EXTRACT) {
// Tesseract may see the t in haystack as a ! some times...
//or it might see dehayslack...
//TODO: figure out how to make this test less hacky
@@ -330,7 +336,7 @@ public class PDFParserTest extends TikaTest {
//TIKA-1990, test that an embedded jpeg is correctly decoded
PDFParserConfig config = new PDFParserConfig();
config.setExtractInlineImages(true);
- config.getOcr().setStrategy(OcrConfig.Strategy.NO_OCR);
+ config.setText(PDFParserConfig.TextPolicy.EXTRACT);
ParseContext context = new ParseContext();
context.set(PDFParserConfig.class, config);
@@ -351,7 +357,7 @@ public class PDFParserTest extends TikaTest {
PDFParserConfig config = new PDFParserConfig();
config.setExtractInlineImages(true);
config.setExtractUniqueInlineImagesOnly(false);
- config.getOcr().setStrategy(OcrConfig.Strategy.NO_OCR);
+ config.setText(PDFParserConfig.TextPolicy.EXTRACT);
context.set(PDFParserConfig.class, config);
@@ -378,7 +384,7 @@ public class PDFParserTest extends TikaTest {
public void testJBIG2OCROnly() throws Exception {
assumeTrue(canRunOCR(), "can't run OCR");
PDFParserConfig config = new PDFParserConfig();
- config.getOcr().setStrategy(OcrConfig.Strategy.OCR_ONLY);
+ config.setText(PDFParserConfig.TextPolicy.OCR);
ParseContext context = new ParseContext();
context.set(PDFParserConfig.class, config);
//make sure everything works with regular xml _and_ with recursive
@@ -390,7 +396,7 @@ public class PDFParserTest extends TikaTest {
public void testJPEG2000() throws Exception {
assumeTrue(canRunOCR(), "can't run OCR");
PDFParserConfig config = new PDFParserConfig();
- config.getOcr().setStrategy(OcrConfig.Strategy.OCR_ONLY);
+ config.setText(PDFParserConfig.TextPolicy.OCR);
ParseContext context = new ParseContext();
context.set(PDFParserConfig.class, config);
//make sure everything works with regular xml _and_ with recursive
@@ -406,13 +412,13 @@ public class PDFParserTest extends TikaTest {
assertContains("Happy New Year", getXML("testOCR.pdf").xml);
PDFParserConfig config = new PDFParserConfig();
- config.getOcr().setStrategy(OcrConfig.Strategy.AUTO);
+ config.setText(PDFParserConfig.TextPolicy.AUTO);
ParseContext context = new ParseContext();
context.set(PDFParserConfig.class, config);
XMLResult xmlResult = getXML("testOCR.pdf", context);
assertContains("Happy New Year", xmlResult.xml);
- config.getOcr().setStrategy(OcrConfig.Strategy.NO_OCR);
+ config.setText(PDFParserConfig.TextPolicy.EXTRACT);
String txt = getText("testOCR.pdf", new Metadata(), context);
assertEquals("", txt.trim());
}
@@ -422,7 +428,7 @@ public class PDFParserTest extends TikaTest {
assumeTrue(canRunOCR(), "can't run OCR");
PDFParserConfig config = new PDFParserConfig();
config.getOcr().setRenderingStrategy(OcrConfig.RenderingStrategy.ALL);
- config.getOcr().setStrategy(OcrConfig.Strategy.OCR_ONLY);
+ config.setText(PDFParserConfig.TextPolicy.OCR);
ParseContext parseContext = new ParseContext();
parseContext.set(PDFParserConfig.class, config);
XMLResult xmlResult = getXML("testPDF_XFA_govdocs1_258578.pdf",
parseContext);
@@ -430,7 +436,7 @@ public class PDFParserTest extends TikaTest {
assertContains("Applications", xmlResult.xml);
config.getOcr().setRenderingStrategy(OcrConfig.RenderingStrategy.NO_TEXT);
- config.getOcr().setStrategy(OcrConfig.Strategy.OCR_ONLY);
+ config.setText(PDFParserConfig.TextPolicy.OCR);
parseContext.set(PDFParserConfig.class, config);
xmlResult = getXML("testPDF_XFA_govdocs1_258578.pdf", parseContext);
assertContains("NATIONAL", xmlResult.xml);
@@ -582,7 +588,7 @@ public class PDFParserTest extends TikaTest {
String json = "{\"pdf-parser\": {\"sortByPosition\": true, " +
"\"extractInlineImages\": true, \"ocr\": {\"strategy\":
\"AUTO\"}}}";
- com.fasterxml.jackson.databind.ObjectMapper mapper =
TikaObjectMapperFactory.getMapper();
+ ObjectMapper mapper = TikaObjectMapperFactory.getMapper();
ParseContext deserialized = mapper.readValue(json, ParseContext.class);
// Verify config was stored as a JSON config entry
@@ -598,7 +604,7 @@ public class PDFParserTest extends TikaTest {
"sortByPosition should be preserved");
assertTrue(deserializedConfig.isExtractInlineImages(),
"extractInlineImages should be preserved");
- assertEquals(OcrConfig.Strategy.AUTO,
deserializedConfig.getOcr().getStrategy(),
+ assertEquals(PDFParserConfig.TextPolicy.AUTO,
deserializedConfig.getText(),
"ocr.strategy should be preserved");
}
@@ -652,12 +658,31 @@ public class PDFParserTest extends TikaTest {
.get(MediaType.application("pdf"));
assertEquals("org.apache.tika.parser.pdf.PDFParser",
pdfParser.getClass().getName());
- assertEquals(OcrConfig.Strategy.OCR_ONLY,
- ((PDFParser)
pdfParser).getPDFParserConfig().getOcr().getStrategy());
+ assertEquals(PDFParserConfig.TextPolicy.OCR,
+ ((PDFParser) pdfParser).getPDFParserConfig().getText());
assertEquals(OcrConfig.ImageType.RGB,
((PDFParser)
pdfParser).getPDFParserConfig().getOcr().getImageType());
}
+ /** A config written with the 4.0 ocr.strategy alias is dumped as the 4.1
"text" key. */
+ @Test
+ public void testOcrStrategyAliasDumpsAsText() throws Exception {
+ TikaLoader loader =
TikaLoaderHelper.getLoader("tika-config-non-primitives.json");
+ loader.loadAutoDetectParser();
+ JsonNode pdf = null;
+ for (JsonNode entry
+ : new ObjectMapper().readTree(loader.toJson())
+ .get("parsers")) {
+ if (entry.has("pdf-parser")) {
+ pdf = entry.get("pdf-parser");
+ }
+ }
+ assertNotNull(pdf);
+ JsonNode config = pdf.has("defaultConfig") ? pdf.get("defaultConfig")
: pdf;
+ assertEquals("OCR", config.get("text").asText());
+ assertFalse(config.get("ocr").has("strategy"), "the alias is not
written back");
+ }
+
private ParseContext configureRenderingParseContext() {
ParseContext parseContext = new ParseContext();
parseContext.set(EmbeddedDocumentExtractor.class, new
RenderCaptureExtractor());
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/resources/config-examples/pdf-parser-full.json
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/resources/config-examples/pdf-parser-full.json
index 85548d6f08..2e1ae9e0d3 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/resources/config-examples/pdf-parser-full.json
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/resources/config-examples/pdf-parser-full.json
@@ -49,6 +49,13 @@
// What the parser releases to the inference bindings (see the
inference page):
// TEXT is the extracted text; PAGES renders every page for a PAGES
binding.
"inference": { "input": ["TEXT"] },
+ // Where a page's text comes from. EXTRACT: the content stream, never
OCR. AUTO: the
+ // content stream, OCR on pages the "strategyAuto" thresholds flag.
EXTRACT_AND_OCR:
+ // both, every page. OCR: the engine only. NONE: no text at all; pages
are still
+ // rendered for annotators and inference. The engine comes from
"text-recognizers".
+ // The 4.0 "ocr": {"strategy": ...} values still load as aliases.
+ // Options: EXTRACT, AUTO, EXTRACT_AND_OCR, OCR, NONE
+ "text": "AUTO",
// Tagged PDFs: follow the structure tree (headings, lists, tables) or
use the plain
// text stripper. AUTO uses the tree on pages that pass a quality
gate; TAGS always;
// NONE never. A PDF without a structure tree always uses the stripper.
@@ -88,11 +95,7 @@
// Which page content to render for OCR.
// Options: NO_TEXT, TEXT_ONLY, VECTOR_GRAPHICS_ONLY, ALL
"renderingStrategy": "ALL",
- // When to run OCR; AUTO runs it only on text-poor pages and
replaces their text
- // with the engine's output. The engine comes from
"text-recognizers".
- // Options: AUTO, NO_OCR, OCR_ONLY, OCR_AND_TEXT_EXTRACTION
- "strategy": "AUTO",
- // Per-page character thresholds that trigger AUTO OCR.
+ // Per-page character thresholds that trigger OCR under "text":
"AUTO".
"strategyAuto": {
"totalCharsPerPage": 10,
"unmappedUnicodeCharsPerPage": 10
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
index 566359a6e2..835192d828 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/AbstractPDF2XHTML.java
@@ -16,10 +16,10 @@
*/
package org.apache.tika.parser.pdf;
-import static org.apache.tika.parser.pdf.OcrConfig.Strategy.AUTO;
-import static org.apache.tika.parser.pdf.OcrConfig.Strategy.NO_OCR;
-import static
org.apache.tika.parser.pdf.OcrConfig.Strategy.OCR_AND_TEXT_EXTRACTION;
-import static org.apache.tika.parser.pdf.OcrConfig.Strategy.OCR_ONLY;
+import static org.apache.tika.parser.pdf.PDFParserConfig.TextPolicy.AUTO;
+import static
org.apache.tika.parser.pdf.PDFParserConfig.TextPolicy.EXTRACT_AND_OCR;
+import static org.apache.tika.parser.pdf.PDFParserConfig.TextPolicy.NONE;
+import static org.apache.tika.parser.pdf.PDFParserConfig.TextPolicy.OCR;
import java.awt.image.BufferedImage;
import java.io.BufferedInputStream;
@@ -257,7 +257,7 @@ class AbstractPDF2XHTML extends PDFTextStripper {
}
}
this.pageAnnotators = annotators;
- if (config.getOcr().getStrategy() == AUTO &&
ContentEnrichers.hasTextRecognizer(
+ if (config.getText() == AUTO && ContentEnrichers.hasTextRecognizer(
contentEnrichers, ocrImageMediaType, renderTarget, context)) {
this.pageBuffer = new PageTextBuffer(handler);
this.xhtml = new XHTMLContentHandler(pageBuffer, metadata,
context);
@@ -356,7 +356,7 @@ class AbstractPDF2XHTML extends PDFTextStripper {
}
List<PageTextBuffer.SaxEvent> captured =
pageBuffer == null ? Collections.emptyList() :
pageBuffer.stop();
- boolean wanted = config.getOcr().getStrategy() == AUTO &&
+ boolean wanted = config.getText() == AUTO &&
ocrWanted(pageBuffer == null ? null : pageBuffer.text());
pageDecision = wanted ? PageText.OCR_WANTED : PageText.KEEP;
if (wanted) {
@@ -691,16 +691,16 @@ class AbstractPDF2XHTML extends PDFTextStripper {
TEXT
}
- PageOcr doOCROnCurrentPage(PDPage pdPage, OcrConfig.Strategy ocrStrategy)
+ PageOcr doOCROnCurrentPage(PDPage pdPage, PDFParserConfig.TextPolicy text)
throws IOException, TikaException, SAXException {
- PageOcr result = dispatchOcr(pdPage, ocrStrategy);
+ PageOcr result = dispatchOcr(pdPage, text);
currentPageOcr = result;
return result;
}
- private PageOcr dispatchOcr(PDPage pdPage, OcrConfig.Strategy ocrStrategy)
+ private PageOcr dispatchOcr(PDPage pdPage, PDFParserConfig.TextPolicy text)
throws IOException, TikaException, SAXException {
- if (ocrStrategy.equals(NO_OCR)) {
+ if (text != AUTO && text != OCR && text != EXTRACT_AND_OCR) {
return PageOcr.SKIPPED;
}
//count the number of times that OCR would have been called
@@ -715,14 +715,14 @@ class AbstractPDF2XHTML extends PDFTextStripper {
return PageOcr.SKIPPED;
}
if (ocrEngine == null) {
- if ((ocrStrategy == OCR_ONLY || ocrStrategy ==
OCR_AND_TEXT_EXTRACTION)
+ if ((text == OCR || text == EXTRACT_AND_OCR)
&& !ContentEnrichers.isDisabled(context)) {
throw new TikaException(
"I regret that I couldn't find an OCR engine to handle
" +
ocrImageMediaType + ". Name one that covers it
in " +
"\"text-recognizers\" (a configured list is
authoritative), " +
"add one to the classpath when no list is
configured, " +
- "or set the OCR strategy to NO_OCR.");
+ "or set \"pdf-parser\": {\"text\":
\"EXTRACT\"}.");
}
return PageOcr.SKIPPED;
}
@@ -984,8 +984,8 @@ class AbstractPDF2XHTML extends PDFTextStripper {
for (PDAnnotation annotation : page.getAnnotations()) {
processPageAnnotation(annotation);
}
- if (config.getOcr().getStrategy() == OCR_AND_TEXT_EXTRACTION) {
- doOCROnCurrentPage(page, OCR_AND_TEXT_EXTRACTION);
+ if (config.getText() == EXTRACT_AND_OCR) {
+ doOCROnCurrentPage(page, EXTRACT_AND_OCR);
} else if (pageDecision == PageText.OCR_WANTED) {
if (doOCROnCurrentPage(page, AUTO) != PageOcr.TEXT) {
replay(pendingText);
@@ -1076,7 +1076,7 @@ class AbstractPDF2XHTML extends PDFTextStripper {
attributes);
}
}
- if (! config.isExtractAnnotationText()) {
+ if (!config.isExtractAnnotationText() || config.getText() == NONE) {
return;
}
// TODO: remove once PDFBOX-1143 is fixed:
@@ -1358,8 +1358,8 @@ class AbstractPDF2XHTML extends PDFTextStripper {
@Override
protected void endDocument(PDDocument pdf) throws IOException {
try {
- // Extract text for any bookmarks:
- if (config.isExtractBookmarksText()) {
+ // NONE writes no text from any source: bookmarks, forms and
annotations included
+ if (config.isExtractBookmarksText() && config.getText() != NONE) {
extractBookmarkText();
}
@@ -1378,7 +1378,7 @@ class AbstractPDF2XHTML extends PDFTextStripper {
extractXMPXFA();
//extract acroform data at end of doc
- if (config.isExtractAcroFormContent() == true) {
+ if (config.isExtractAcroFormContent() && config.getText() != NONE)
{
try {
extractAcroForm(pdf);
} catch (IOException e) {
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/OCR2XHTML.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/OCR2XHTML.java
index c3de03490b..fc26f0aac6 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/OCR2XHTML.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/OCR2XHTML.java
@@ -97,7 +97,10 @@ class OCR2XHTML extends AbstractPDF2XHTML {
public void processPage(PDPage pdPage) throws IOException {
try {
startPage(pdPage);
- doOCROnCurrentPage(pdPage, OcrConfig.Strategy.OCR_ONLY);
+ // NONE walks the pages for their renders and writes no text
+ if (config.getText() != PDFParserConfig.TextPolicy.NONE) {
+ doOCROnCurrentPage(pdPage, PDFParserConfig.TextPolicy.OCR);
+ }
// endPage annotates the page when no engine ran, and offers it to
inference
endPage(pdPage);
} catch (TikaException | SAXException e) {
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/OcrConfig.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/OcrConfig.java
index 0b54b7b5a2..6ccf9755e0 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/OcrConfig.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/OcrConfig.java
@@ -27,11 +27,28 @@ public class OcrConfig implements Serializable {
private static final long serialVersionUID = 1L;
+ /**
+ * The 4.0 spelling of {@link PDFParserConfig.TextPolicy}, kept so {@code
ocr.strategy}
+ * still loads.
+ *
+ * @deprecated since 4.1.0; set {@code "text"} on the parser instead.
+ */
+ @Deprecated
public enum Strategy {
- AUTO,
- NO_OCR,
- OCR_ONLY,
- OCR_AND_TEXT_EXTRACTION
+ AUTO(PDFParserConfig.TextPolicy.AUTO),
+ NO_OCR(PDFParserConfig.TextPolicy.EXTRACT),
+ OCR_ONLY(PDFParserConfig.TextPolicy.OCR),
+ OCR_AND_TEXT_EXTRACTION(PDFParserConfig.TextPolicy.EXTRACT_AND_OCR);
+
+ private final PDFParserConfig.TextPolicy text;
+
+ Strategy(PDFParserConfig.TextPolicy text) {
+ this.text = text;
+ }
+
+ public PDFParserConfig.TextPolicy toText() {
+ return text;
+ }
}
public enum RenderingStrategy {
@@ -115,7 +132,8 @@ public class OcrConfig implements Serializable {
}
}
- private Strategy strategy = Strategy.AUTO;
+ /** The alias, null unless a 4.0 config set it; no getter so a dump writes
"text" only. */
+ private Strategy strategy;
private StrategyAuto strategyAuto = StrategyAuto.BETTER;
private RenderingStrategy renderingStrategy = RenderingStrategy.ALL;
private int dpi = 300;
@@ -153,14 +171,19 @@ public class OcrConfig implements Serializable {
*/
private int maxPagesToOcr = -1;
- public Strategy getStrategy() {
- return strategy;
- }
-
+ /**
+ * @deprecated since 4.1.0; use {@link PDFParserConfig#setText}. Ignored
when
+ * {@code "text"} is set.
+ */
+ @Deprecated
public void setStrategy(Strategy strategy) {
this.strategy = strategy;
}
+ PDFParserConfig.TextPolicy legacyText() {
+ return strategy == null ? null : strategy.toText();
+ }
+
public StrategyAuto getStrategyAuto() {
return strategyAuto;
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
index 6d52af77d4..4572bd48d5 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java
@@ -221,8 +221,8 @@ public class PDFParser implements Parser, RenderingParser,
EnrichingParser {
if (handler != null) {
if (shouldHandleXFAOnly(hasXFA, localConfig)) {
handleXFAOnly(pdfDocument, handler, metadata, context);
- } else if (localConfig.getOcr().getStrategy()
- .equals(OcrConfig.Strategy.OCR_ONLY)) {
+ } else if (localConfig.getText() ==
PDFParserConfig.TextPolicy.OCR
+ || localConfig.getText() ==
PDFParserConfig.TextPolicy.NONE) {
OCR2XHTML.process(pdfDocument, handler, context, metadata,
localConfig, renderer, contentEnrichers);
} else if (hasMarkedContent &&
localConfig.getMarkedContent().getStrategy()
@@ -443,7 +443,8 @@ public class PDFParser implements Parser, RenderingParser,
EnrichingParser {
return true;
}
- if (localConfig.getOcr().getStrategy() == OcrConfig.Strategy.NO_OCR) {
+ if (localConfig.getText() == PDFParserConfig.TextPolicy.EXTRACT
+ || localConfig.getText() == PDFParserConfig.TextPolicy.NONE) {
return false;
}
//TODO: test that this is not AUTO with no OCR parser installed
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
index d5607ef47f..d7948bef97 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParserConfig.java
@@ -39,6 +39,25 @@ public class PDFParserConfig implements Serializable {
/**
* Mode for checking document access permissions.
*/
+ /**
+ * Where a page's text comes from: the content stream, OCR of the rendered
page, both,
+ * the per-page verdict, or nowhere. Set with {@code "text"}; the 4.0
{@code ocr.strategy}
+ * spellings are aliases ({@code NO_OCR} = {@link #EXTRACT}, {@code
OCR_ONLY} =
+ * {@link #OCR}, {@code OCR_AND_TEXT_EXTRACTION} = {@link
#EXTRACT_AND_OCR}).
+ */
+ public enum TextPolicy {
+ /** The content stream only; never OCR. */
+ EXTRACT,
+ /** The content stream, OCR where the verdict says the page needs it.
*/
+ AUTO,
+ /** Both, every page. */
+ EXTRACT_AND_OCR,
+ /** OCR only; the content stream is not read. */
+ OCR,
+ /** No text from any source; pages are still rendered for annotators
and inference. */
+ NONE
+ }
+
public enum AccessCheckMode {
/**
* Don't check extraction permissions. Content will always be extracted
@@ -116,6 +135,8 @@ public class PDFParserConfig implements Serializable {
private boolean ifXFAExtractOnlyXFA = false;
private OcrConfig ocr = new OcrConfig();
+ /** Null until set; {@link #getText()} then falls back to the {@code
ocr.strategy} alias. */
+ private TextPolicy text;
private InferenceConfig inference = new InferenceConfig();
@@ -531,6 +552,19 @@ public class PDFParserConfig implements Serializable {
this.ocr = ocr;
}
+ /** The text policy: {@code "text"} if set, else the {@code ocr.strategy}
alias, else AUTO. */
+ public TextPolicy getText() {
+ if (text != null) {
+ return text;
+ }
+ TextPolicy legacy = ocr == null ? null : ocr.legacyText();
+ return legacy != null ? legacy : TextPolicy.AUTO;
+ }
+
+ public void setText(TextPolicy text) {
+ this.text = text;
+ }
+
/** What this parser releases to the inference bindings. */
public InferenceConfig getInference() {
return inference;
@@ -542,7 +576,7 @@ public class PDFParserConfig implements Serializable {
// OCR settings are configured through the nested OcrConfig
(getOcr()/setOcr()).
// The flat ocr* convenience accessors (getOcrStrategy/setOcrDPI/...) were
removed in
- // 4.x so that "ocr" is the single JSON spelling; use
getOcr().setStrategy(...) etc.
+ // 4.x so that "ocr" is the single JSON spelling; use getOcr().setDpi(...)
etc.
/**
* @return whether or not to extract PDActions
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFMarkedContentGateTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFMarkedContentGateTest.java
index 25bd1dab3e..a6874d14ca 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFMarkedContentGateTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFMarkedContentGateTest.java
@@ -1089,7 +1089,7 @@ public class PDFMarkedContentGateTest extends TikaTest {
finish(cs);
PDFParserConfig config = config(MarkedContentConfig.Strategy.TAGS);
- config.getOcr().setStrategy(OcrConfig.Strategy.AUTO);
+ config.setText(PDFParserConfig.TextPolicy.AUTO);
config.getOcr().setStrategyAuto(new OcrConfig.StrategyAuto(0.02f,
10));
Result result = parse(b.bytes(), config, mockOcrParser(config,
"MOCK_OCR_CONTENT"));
assertContains("<p>Enough tagged text", result.page(1));
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
index d25c4e856e..ddcd9eb5c6 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/test/java/org/apache/tika/parser/pdf/PDFParserTest.java
@@ -1281,7 +1281,7 @@ public class PDFParserTest extends TikaTest {
//behavior
config = new PDFParserConfig();
config.getOcr().setDpi(10000);
- config.getOcr().setStrategy(OcrConfig.Strategy.NO_OCR);
+ config.setText(PDFParserConfig.TextPolicy.EXTRACT);
pc.set(PDFParserConfig.class, config);
text = getText("testPDFTwoTextBoxes.pdf", p, new Metadata(), pc);
text = text.replaceAll("\\s+", " ");
@@ -1866,7 +1866,7 @@ public class PDFParserTest extends TikaTest {
parser.setContentEnrichers(enrichers);
EnrichingImageParser imageParser = new EnrichingImageParser(enrichers);
PDFParserConfig config = new PDFParserConfig();
- config.getOcr().setStrategy(OcrConfig.Strategy.NO_OCR);
+ config.setText(PDFParserConfig.TextPolicy.EXTRACT);
config.setImageStrategy(PDFParserConfig.IMAGE_STRATEGY.RENDER_PAGES_AT_PAGE_END);
config.getOcr().setDpi(20);
ParseContext context = new ParseContext();
@@ -1893,7 +1893,7 @@ public class PDFParserTest extends TikaTest {
PDFParser parser = new PDFParser();
parser.setContentEnrichers(enrichers);
PDFParserConfig config = new PDFParserConfig();
-
config.getOcr().setStrategy(OcrConfig.Strategy.OCR_AND_TEXT_EXTRACTION);
+ config.setText(PDFParserConfig.TextPolicy.EXTRACT_AND_OCR);
config.getOcr().setMaxPagesToOcr(1);
config.setImageStrategy(PDFParserConfig.IMAGE_STRATEGY.RENDER_PAGES_AT_PAGE_END);
config.getOcr().setDpi(20);
@@ -1919,7 +1919,7 @@ public class PDFParserTest extends TikaTest {
PDFParser parser = new PDFParser();
parser.setContentEnrichers(enrichers);
PDFParserConfig config = new PDFParserConfig();
- config.getOcr().setStrategy(OcrConfig.Strategy.OCR_ONLY);
+ config.setText(PDFParserConfig.TextPolicy.OCR);
config.getOcr().setMaxPagesToOcr(1);
config.setImageStrategy(PDFParserConfig.IMAGE_STRATEGY.RENDER_PAGES_AT_PAGE_END);
config.getOcr().setDpi(20);
@@ -1965,9 +1965,9 @@ public class PDFParserTest extends TikaTest {
return adp;
}
- private static PDFParserConfig pagesForInference(OcrConfig.Strategy
strategy) {
+ private static PDFParserConfig
pagesForInference(PDFParserConfig.TextPolicy strategy) {
PDFParserConfig config = new PDFParserConfig();
- config.getOcr().setStrategy(strategy);
+ config.setText(strategy);
config.getOcr().setDpi(20);
config.getInference().setInput(List.of(InferenceConfig.Input.PAGES));
return config;
@@ -1976,8 +1976,8 @@ public class PDFParserTest extends TikaTest {
/** One render per page feeds OCR and inference alike; every page is
offered once. */
@Test
public void testPagesOfferedOncePerPageFromOneRender() throws Exception {
- for (OcrConfig.Strategy strategy :
List.of(OcrConfig.Strategy.OCR_AND_TEXT_EXTRACTION,
- OcrConfig.Strategy.OCR_ONLY, OcrConfig.Strategy.NO_OCR)) {
+ for (PDFParserConfig.TextPolicy strategy :
List.of(PDFParserConfig.TextPolicy.EXTRACT_AND_OCR,
+ PDFParserConfig.TextPolicy.OCR,
PDFParserConfig.TextPolicy.EXTRACT)) {
PageEnricher recognizer = new PageEnricher("MOCK_OCR_CONTENT",
true);
PDFParser parser = new PDFParser();
parser.setContentEnrichers(new
CompositeContentEnricher(List.of(recognizer)));
@@ -1996,7 +1996,7 @@ public class PDFParserTest extends TikaTest {
assertSame(metadata, hook.document, label + ": the pages belong to
the PDF");
assertTrue(hook.sizes.get(0) > 0, label + ": the render has
bytes");
assertEquals(2, tracker.getNextId() - 1, label + ": one render per
page");
- assertEquals(strategy == OcrConfig.Strategy.NO_OCR ? 0 : 2,
recognizer.calls, label);
+ assertEquals(strategy == PDFParserConfig.TextPolicy.EXTRACT ? 0 :
2, recognizer.calls, label);
assertArrayEquals(new String[]{"PAGES"},
metadata.getValues(TikaCoreProperties.INFERENCE_RELEASED),
label + ": the PDF says it released its pages, not its
text");
@@ -2009,7 +2009,7 @@ public class PDFParserTest extends TikaTest {
PageHook unasked = new PageHook();
ParseContext context = new ParseContext();
PDFParserConfig config = new PDFParserConfig();
- config.getOcr().setStrategy(OcrConfig.Strategy.NO_OCR);
+ config.setText(PDFParserConfig.TextPolicy.EXTRACT);
context.set(PDFParserConfig.class, config);
RenderingTracker tracker = new RenderingTracker();
context.set(RenderingTracker.class, tracker);
@@ -2022,7 +2022,7 @@ public class PDFParserTest extends TikaTest {
PageHook unwilling = new PageHook();
context = new ParseContext();
- context.set(PDFParserConfig.class,
pagesForInference(OcrConfig.Strategy.NO_OCR));
+ context.set(PDFParserConfig.class,
pagesForInference(PDFParserConfig.TextPolicy.EXTRACT));
context.set(RenderingTracker.class, tracker);
try (TikaInputStream tis =
getResourceAsStream("/test-documents/testPDF_bookmarks.pdf")) {
hooked(new PDFParser(), unwilling).parse(tis, new
ToXMLContentHandler(),
@@ -2033,6 +2033,50 @@ public class PDFParserTest extends TikaTest {
assertEquals(0, tracker.getNextId() - 1, "no binding wants pages:
nothing is rendered");
}
+ /** NONE: no text from the stream or an engine, no engine complaint, pages
still offered. */
+ @Test
+ public void testTextNoneRendersForInferenceAndWritesNoText() throws
Exception {
+ PageEnricher recognizer = new PageEnricher("MOCK_OCR_CONTENT", true);
+ PDFParser parser = new PDFParser();
+ parser.setContentEnrichers(new
CompositeContentEnricher(List.of(recognizer)));
+ PageHook hook = new PageHook();
+ hook.wantsPages = true;
+ ParseContext context = new ParseContext();
+ context.set(PDFParserConfig.class,
pagesForInference(PDFParserConfig.TextPolicy.NONE));
+ Metadata metadata = new Metadata();
+ ToXMLContentHandler handler = new ToXMLContentHandler();
+ try (TikaInputStream tis =
getResourceAsStream("/test-documents/testPDF_bookmarks.pdf")) {
+ hooked(parser, hook).parse(tis, handler, metadata, context);
+ }
+ assertEquals(List.of(1, 2), hook.pages, "pages are rendered and
offered");
+ assertEquals(0, recognizer.calls, "no OCR");
+ String xml = handler.toString();
+ assertNotContained("MOCK_OCR_CONTENT", xml);
+ assertNotContained("Denmark", xml);
+ assertEquals("2", metadata.get(PagedText.N_PAGES), "metadata is still
extracted");
+
+ // no engine at all: NONE never asks for one
+ context = new ParseContext();
+ context.set(PDFParserConfig.class,
pagesForInference(PDFParserConfig.TextPolicy.NONE));
+ xml = parsePdfToXml(new PDFParser(), "testPDF_bookmarks.pdf", new
Metadata(), context);
+ assertNotContained("Denmark", xml);
+ }
+
+ /** The 4.0 ocr.strategy spellings map onto "text"; an explicit "text"
wins. */
+ @Test
+ public void testOcrStrategyAlias() throws Exception {
+ PDFParserConfig config = new PDFParserConfig();
+ assertEquals(PDFParserConfig.TextPolicy.AUTO, config.getText(),
"default");
+ config.getOcr().setStrategy(OcrConfig.Strategy.NO_OCR);
+ assertEquals(PDFParserConfig.TextPolicy.EXTRACT, config.getText());
+ config.getOcr().setStrategy(OcrConfig.Strategy.OCR_ONLY);
+ assertEquals(PDFParserConfig.TextPolicy.OCR, config.getText());
+
config.getOcr().setStrategy(OcrConfig.Strategy.OCR_AND_TEXT_EXTRACTION);
+ assertEquals(PDFParserConfig.TextPolicy.EXTRACT_AND_OCR,
config.getText());
+ config.setText(PDFParserConfig.TextPolicy.NONE);
+ assertEquals(PDFParserConfig.TextPolicy.NONE, config.getText(), "text
wins over the alias");
+ }
+
/** The request's switch: no OCR, text kept, and OCR_ONLY does not
complain about an engine. */
@Test
public void testRecognizersOffPerRequest() throws Exception {
@@ -2052,7 +2096,7 @@ public class PDFParserTest extends TikaTest {
assertContains("42936", xml);
PDFParserConfig ocrOnly = new PDFParserConfig();
- ocrOnly.getOcr().setStrategy(OcrConfig.Strategy.OCR_ONLY);
+ ocrOnly.setText(PDFParserConfig.TextPolicy.OCR);
context = new ParseContext();
context.set(PDFParserConfig.class, ocrOnly);
context.set(TextRecognizerSelection.class, off);
@@ -2245,7 +2289,7 @@ public class PDFParserTest extends TikaTest {
@Test
public void testOCRPageTimeoutDoesNotAbortWholeDocument() throws Exception
{
PDFParserConfig config = new PDFParserConfig();
- config.getOcr().setStrategy(OcrConfig.Strategy.OCR_ONLY);
+ config.setText(PDFParserConfig.TextPolicy.OCR);
ParseContext context = new ParseContext();
context.set(PDFParserConfig.class, config);
@@ -2329,7 +2373,7 @@ public class PDFParserTest extends TikaTest {
@Test
public void testChunksFromAllOcrPagesReachParent() throws Exception {
PDFParserConfig config = new PDFParserConfig();
- config.getOcr().setStrategy(OcrConfig.Strategy.OCR_ONLY);
+ config.setText(PDFParserConfig.TextPolicy.OCR);
ParseContext context = new ParseContext();
context.set(PDFParserConfig.class, config);
@@ -2369,7 +2413,7 @@ public class PDFParserTest extends TikaTest {
@Test
public void testExplicitContentEnricherReceivesRenderedPages() throws
Exception {
PDFParserConfig config = new PDFParserConfig();
- config.getOcr().setStrategy(OcrConfig.Strategy.OCR_ONLY);
+ config.setText(PDFParserConfig.TextPolicy.OCR);
ParseContext context = new ParseContext();
context.set(PDFParserConfig.class, config);
diff --git
a/tika-serialization/src/main/java/org/apache/tika/config/loader/PresetRegistry.java
b/tika-serialization/src/main/java/org/apache/tika/config/loader/PresetRegistry.java
index 5d9cc33459..b2245a9424 100644
---
a/tika-serialization/src/main/java/org/apache/tika/config/loader/PresetRegistry.java
+++
b/tika-serialization/src/main/java/org/apache/tika/config/loader/PresetRegistry.java
@@ -57,7 +57,7 @@ import
org.apache.tika.serialization.serdes.ParseContextDeserializer;
* <pre>
* "presets": {
* "some-catalog-preset": true,
- * "ocr-heavy": { "pdf-parser": { "ocr": { "strategy":
"OCR_AND_TEXT_EXTRACTION" } } }
+ * "ocr-heavy": { "pdf-parser": { "text": "EXTRACT_AND_OCR" } }
* }
* </pre>
*
diff --git
a/tika-serialization/src/main/java/org/apache/tika/config/loader/TikaLoader.java
b/tika-serialization/src/main/java/org/apache/tika/config/loader/TikaLoader.java
index 13d9d55bc6..189fd919cf 100644
---
a/tika-serialization/src/main/java/org/apache/tika/config/loader/TikaLoader.java
+++
b/tika-serialization/src/main/java/org/apache/tika/config/loader/TikaLoader.java
@@ -88,7 +88,7 @@ import org.apache.tika.utils.StringUtils;
* "pdf-parser": {
* "_mime-include": ["application/pdf"],
* "_mime-exclude": ["application/pdf+fdf"],
- * "ocr": {"strategy": "AUTO"},
+ * "text": "AUTO",
* "extractInlineImages": true
* }
* }
@@ -478,7 +478,7 @@ public class TikaLoader {
*
* // At runtime, create per-request overrides
* PDFParserConfig requestConfig = new PDFParserConfig();
- * requestConfig.getOcr().setStrategy(OcrConfig.Strategy.NO_OCR);
+ * requestConfig.setText(PDFParserConfig.TextPolicy.EXTRACT);
*
* // Merge: base config values + request overrides
* // (Note: for runtime merging, use JsonMergeUtils directly or
loadConfig on a runtime loader)
diff --git
a/tika-server/docker-build/sample-configs/customocr/tika-config-rendered.json
b/tika-server/docker-build/sample-configs/customocr/tika-config-rendered.json
index 792de50573..422f3413b4 100644
---
a/tika-server/docker-build/sample-configs/customocr/tika-config-rendered.json
+++
b/tika-server/docker-build/sample-configs/customocr/tika-config-rendered.json
@@ -8,8 +8,8 @@
{ "default-parser": {} },
{
"pdf-parser": {
+ "text": "OCR",
"ocr": {
- "strategy": "ocr_only",
"imageType": "rgb",
"dpi": 100
}
diff --git
a/tika-server/tika-server-core/src/test/resources/config-examples/server-with-parsers.json
b/tika-server/tika-server-core/src/test/resources/config-examples/server-with-parsers.json
index dc1bc58b80..e97f667bcb 100644
---
a/tika-server/tika-server-core/src/test/resources/config-examples/server-with-parsers.json
+++
b/tika-server/tika-server-core/src/test/resources/config-examples/server-with-parsers.json
@@ -7,9 +7,7 @@
{
"pdf-parser": {
"extractInlineImages": true,
- "ocr": {
- "strategy": "AUTO"
- }
+ "text": "AUTO"
}
},
{
diff --git
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/InferenceBindingsWireTest.java
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/InferenceBindingsWireTest.java
index e6a7bfb224..9d1b83ae2a 100644
---
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/InferenceBindingsWireTest.java
+++
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/InferenceBindingsWireTest.java
@@ -108,7 +108,7 @@ public class InferenceBindingsWireTest extends CXFTestBase {
public void testPagesPerRequest() throws Exception {
String config = """
{ "parse-context": {
- "pdf-parser": { "ocr": { "strategy": "NO_OCR", "dpi": 20 },
+ "pdf-parser": { "text": "EXTRACT", "ocr": { "dpi": 20 },
"inference": { "input": ["PAGES"] } },
"inference": { "bindings": ["mock-pages"] } } }
""";
