This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4809-stage-9 in repository https://gitbox.apache.org/repos/asf/tika.git
commit 78fc9e06d5683098cc09fb07b23d137dc7e24b38 Author: tallison <[email protected]> AuthorDate: Mon Aug 10 21:46:32 2026 -0400 TIKA-4809: Fix /tika handler selection -- body-only text, markdown default, no silent fallback --- .../migration-to-4x/migrating-tika-server-4x.adoc | 20 +++++++++++ .../tika/sax/BasicContentHandlerFactory.java | 20 +++++++---- .../tika/server/core/resource/TikaResource.java | 41 ++++++++++++++++++---- .../apache/tika/server/core/TikaResourceTest.java | 36 +++++++++++++++++++ .../test-documents/mock/hello_world_heading.xml | 27 ++++++++++++++ .../standard/RecursiveMetadataResourceTest.java | 25 +++---------- .../tika/server/standard/TikaResourceTest.java | 24 +++++++++++++ 7 files changed, 161 insertions(+), 32 deletions(-) diff --git a/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc b/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc index e250910342..9c00ba4531 100644 --- a/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc +++ b/docs/modules/ROOT/pages/migration-to-4x/migrating-tika-server-4x.adoc @@ -109,6 +109,26 @@ Collapsing them also removes the `/detect/stream` vs `/detectors` near-collision **Migration:** drop the suffix. `PUT /detect/stream` becomes `PUT /detect`; `PUT /language/stream` and `PUT /language/string` both become `PUT /language`. Request bodies, headers, and responses are unchanged. +=== Handler-Type Changes on `/tika` + +==== `/tika/text` is body-only again + +3.x served this from a `BodyContentHandler`. 4.0.0 prereleases ran it over the whole XHTML document, so output began with the document title as bare text. This is restored to 3.x behaviour: `/tika/text` returns body content only. + +**Migration:** if you relied on the prerelease behaviour, `PUT /tika/json/text` still runs the whole-document handler and returns the content in a JSON envelope. + +==== `/tika/json` and `/tika/config/json` default to markdown + +Previously they hardcoded plain text. With no handler named in the path they now use the server default, which is markdown -- matching `/rmeta`. Name one explicitly to pin it: `/tika/json/text`, `/tika/json/html`, `/tika/json/xml`, `/tika/json/body`, `/tika/json/md`. + +==== Unrecognized handler names are rejected + +`/rmeta/txet` used to fall back to the default handler and return output that looked correct. Any unrecognized handler name in the path is now a 400 naming the valid types (`text`, `txt`, `html`, `xml`, `body`, `markdown`, `md`, `ignore`). This applies to `/tika/json/{handler}` and the `/rmeta/{handler}` family. + +==== New: `POST /tika/config/json/{handler}` + +The PUT family had `/tika/json/{handler}` but the multipart POST family had only fixed paths, so a POST caller wanting text-in-JSON had nowhere to go -- `/tika/config/text` returns raw text with no metadata envelope. Added for parity. Requires `allowPerRequestConfig`. + === Error Response Bodies Are Now JSON In 3.x, error responses from `/tika`, `/rmeta`, and `/unpack` returned a plain-text diff --git a/tika-core/src/main/java/org/apache/tika/sax/BasicContentHandlerFactory.java b/tika-core/src/main/java/org/apache/tika/sax/BasicContentHandlerFactory.java index cc25fecbdf..09d7730a8d 100644 --- a/tika-core/src/main/java/org/apache/tika/sax/BasicContentHandlerFactory.java +++ b/tika-core/src/main/java/org/apache/tika/sax/BasicContentHandlerFactory.java @@ -117,15 +117,19 @@ public class BasicContentHandlerFactory implements StreamingContentHandlerFactor limits.isThrowOnWriteLimit(), context); } + /** Accepted spellings, for error messages. */ + public static final String VALID_HANDLER_TYPE_NAMES = + "text, txt, html, xml, body, markdown, md, ignore"; + /** - * Tries to parse string into handler type. Returns default if string is null or - * parse fails. + * Parses a string into a handler type. * <p/> - * Options: xml, html, text, body, ignore (no content), markdown/md + * Options: xml, html, text/txt, body, ignore (no content), markdown/md * - * @param handlerTypeName string to parse - * @param defaultType type to return if parse fails + * @param handlerTypeName string to parse; null means "unspecified", which yields defaultType + * @param defaultType type to return when handlerTypeName is null * @return handler type + * @throws IllegalArgumentException if handlerTypeName is non-null and not recognized */ public static HANDLER_TYPE parseHandlerType(String handlerTypeName, HANDLER_TYPE defaultType) { if (handlerTypeName == null) { @@ -150,7 +154,11 @@ public class BasicContentHandlerFactory implements StreamingContentHandlerFactor case "md": return HANDLER_TYPE.MARKDOWN; default: - return defaultType; + // A name we don't know is a caller mistake, not a request for the default: + // silently substituting one meant "/rmeta/txet" quietly returned the default + // handler's output and looked like it worked. + throw new IllegalArgumentException("Unrecognized handler type '" + handlerTypeName + + "'. Valid types: " + VALID_HANDLER_TYPE_NAMES); } } diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java index 023045cd1f..a6045dd299 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java @@ -30,6 +30,7 @@ import java.util.Map; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.ws.rs.BadRequestException; import jakarta.ws.rs.Consumes; import jakarta.ws.rs.GET; import jakarta.ws.rs.POST; @@ -405,8 +406,13 @@ public class TikaResource { * @param handlerTypeName the handler type name (text, html, xml, ignore), may be null for default */ public static void setupContentHandlerFactory(ParseContext context, String handlerTypeName) { - BasicContentHandlerFactory.HANDLER_TYPE type = BasicContentHandlerFactory.parseHandlerType( - handlerTypeName, DEFAULT_HANDLER_TYPE); + BasicContentHandlerFactory.HANDLER_TYPE type; + try { + type = BasicContentHandlerFactory.parseHandlerType(handlerTypeName, DEFAULT_HANDLER_TYPE); + } catch (IllegalArgumentException e) { + // The name comes from the URL path, so this is the caller's typo, not our failure. + throw new BadRequestException(e.getMessage()); + } context.set(ContentHandlerFactory.class, BasicContentHandlerFactory.newInstance(type, context)); } @@ -460,7 +466,9 @@ public class TikaResource { TikaInputStream tis = TikaInputStream.get(is); tis.getPath(); // Spool to temp file for pipes-based parsing ParseContext context = createParseContext(); - return produceRawOutput(tis, Metadata.newInstance(context), httpHeaders.getRequestHeaders(), "text"); + // "body", not "text": 3.x served this from BodyContentHandler, so TEXT (whole XHTML + // document, title included as characters) was a 4.x regression. + return produceRawOutput(tis, Metadata.newInstance(context), httpHeaders.getRequestHeaders(), "body"); } /** @@ -520,7 +528,8 @@ public class TikaResource { TikaInputStream tis = TikaInputStream.get(is); tis.getPath(); // Spool to temp file for pipes-based parsing ParseContext context = createParseContext(); - return produceJson(tis, Metadata.newInstance(context), httpHeaders.getRequestHeaders(), "text"); + // null, not "text": no handler was named, so this takes DEFAULT_HANDLER_TYPE. + return produceJson(tis, Metadata.newInstance(context), httpHeaders.getRequestHeaders(), null); } /** @@ -587,7 +596,7 @@ public class TikaResource { ParseContext context = createParseContext(); Metadata metadata = Metadata.newInstance(context); TikaInputStream tis = setupMultipartConfig(attachments, metadata, context); - return produceRawOutput(tis, metadata, context, "text"); + return produceRawOutput(tis, metadata, context, "body"); } /** @@ -677,7 +686,27 @@ public class TikaResource { ParseContext context = createParseContext(); Metadata metadata = Metadata.newInstance(context); TikaInputStream tis = setupMultipartConfig(attachments, metadata, context); - return produceJson(tis, metadata, context, "text"); + return produceJson(tis, metadata, context, null); + } + + /** + * Multipart sibling of {@code PUT /tika/json/{handlerType}}. Without it, a POST caller + * who wants text-in-JSON has nowhere to go: /tika/config/text returns raw text with no + * metadata envelope. + * + * @param handlerTypeName content handler type: text, html, xml, body, markdown, ignore + */ + @POST + @Consumes("multipart/form-data") + @Produces("application/json") + @Path("config/json/{" + HANDLER_TYPE_PARAM + "}") + public Metadata postJsonWithHandler(List<Attachment> attachments, @Context HttpHeaders httpHeaders, + @PathParam(HANDLER_TYPE_PARAM) String handlerTypeName) + throws IOException, TikaConfigException { + ParseContext context = createParseContext(); + Metadata metadata = Metadata.newInstance(context); + TikaInputStream tis = setupMultipartConfig(attachments, metadata, context); + return produceJson(tis, metadata, context, handlerTypeName); } // ==================== Internal methods ==================== diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaResourceTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaResourceTest.java index d6f7f581a9..3e5151bc20 100644 --- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaResourceTest.java +++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaResourceTest.java @@ -43,6 +43,7 @@ public class TikaResourceTest extends CXFTestBase { public static final String TEST_HELLO_WORLD = "test-documents/mock/hello_world.xml"; public static final String TEST_HELLO_WORLD_LONG = "test-documents/mock/hello_world_long.xml"; + public static final String TEST_HELLO_WORLD_HEADING = "test-documents/mock/hello_world_heading.xml"; public static final String TEST_NULL_POINTER = "test-documents/mock/null_pointer.xml"; private static final String TIKA_PATH = "/tika"; @@ -218,4 +219,39 @@ public class TikaResourceTest extends CXFTestBase { assertEquals("my-report.xml", metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY)); } + + /** An unspecified handler takes the default, which is markdown, not text. */ + @Test + public void testJsonDefaultsToMarkdown() throws Exception { + String dflt = jsonContent("/json", TEST_HELLO_WORLD_HEADING); + String md = jsonContent("/json/md", TEST_HELLO_WORLD_HEADING); + String text = jsonContent("/json/text", TEST_HELLO_WORLD_HEADING); + + assertContains("# Chapter One", md); + assertNotFound("# Chapter One", text); + assertEquals(md, dflt, "/tika/json with no handler must match /tika/json/md"); + } + + /** A handler name we don't recognize is the caller's typo: 400, not a silent default. */ + @Test + public void testUnknownHandlerNameIsRejected() throws Exception { + Response response = WebClient + .create(endPoint + TIKA_PATH + "/json/txet") + .put(ClassLoader.getSystemResourceAsStream(TEST_HELLO_WORLD)); + assertEquals(400, response.getStatus()); + } + + private String jsonContent(String path, String doc) throws Exception { + Response response = WebClient + .create(endPoint + TIKA_PATH + path) + .accept("application/json") + .put(ClassLoader.getSystemResourceAsStream(doc)); + assertEquals(200, response.getStatus(), path + " should have succeeded"); + Metadata metadata = JsonMetadata.fromJson(new InputStreamReader( + (InputStream) response.getEntity(), StandardCharsets.UTF_8)); + String content = metadata.get(TikaCoreProperties.TIKA_CONTENT); + return content == null ? "" : content.trim(); + } + + } diff --git a/tika-server/tika-server-core/src/test/resources/test-documents/mock/hello_world_heading.xml b/tika-server/tika-server-core/src/test/resources/test-documents/mock/hello_world_heading.xml new file mode 100644 index 0000000000..bf83bf7879 --- /dev/null +++ b/tika-server/tika-server-core/src/test/resources/test-documents/mock/hello_world_heading.xml @@ -0,0 +1,27 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +<!-- A heading so markdown output is distinguishable from plain text: "# " vs bare text. + Without one, md and text render identically and a default-handler test cannot fail. --> +<mock> + <metadata action="add" name="title">你好，世界</metadata> + <write element="h1">Chapter One</write> + <write element="p">hello world</write> +</mock> diff --git a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/RecursiveMetadataResourceTest.java b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/RecursiveMetadataResourceTest.java index bd37132d93..9d8523a7bc 100644 --- a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/RecursiveMetadataResourceTest.java +++ b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/RecursiveMetadataResourceTest.java @@ -262,20 +262,13 @@ public class RecursiveMetadataResourceTest extends CXFTestBase { assertFalse(content.startsWith("<html")); assertContains("plundered our seas", content); - //unparseable + //an unrecognized handler name is a caller typo: 400, not a silent fallback to the + //default handler, which returned plausible output and looked like it had worked response = WebClient .create(endPoint + META_PATH + UNPARSEABLE_PATH) .accept("application/json") .put(ClassLoader.getSystemResourceAsStream(TEST_RECURSIVE_DOC)); - reader = new InputStreamReader((InputStream) response.getEntity(), UTF_8); - metadataList = JsonMetadataList.fromJson(reader); - assertEquals(12, metadataList.size()); - content = metadataList - .get(6) - .get(TikaCoreProperties.TIKA_CONTENT) - .trim(); - assertFalse(content.startsWith("<html")); - assertContains("plundered our seas", content); + assertEquals(400, response.getStatus()); //xml response = WebClient @@ -357,7 +350,7 @@ public class RecursiveMetadataResourceTest extends CXFTestBase { assertFalse(content.startsWith("<html")); assertContains("plundered our seas", content); - //unparseable + //unrecognized handler name -> 400, same as the PUT family attachmentPart = new Attachment("myworddocx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", ClassLoader.getSystemResourceAsStream(TEST_RECURSIVE_DOC)); webClient = WebClient.create(endPoint + META_PATH + FORM_PATH + UNPARSEABLE_PATH); @@ -366,15 +359,7 @@ public class RecursiveMetadataResourceTest extends CXFTestBase { .type("multipart/form-data") .accept("application/json") .post(attachmentPart); - reader = new InputStreamReader((InputStream) response.getEntity(), UTF_8); - metadataList = JsonMetadataList.fromJson(reader); - assertEquals(12, metadataList.size()); - content = metadataList - .get(6) - .get(TikaCoreProperties.TIKA_CONTENT) - .trim(); - assertFalse(content.startsWith("<html")); - assertContains("plundered our seas", content); + assertEquals(400, response.getStatus()); //xml attachmentPart = diff --git a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaResourceTest.java b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaResourceTest.java index b74634b17c..a7bc3666c1 100644 --- a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaResourceTest.java +++ b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaResourceTest.java @@ -645,4 +645,28 @@ public class TikaResourceTest extends CXFTestBase { TikaTest.assertContains("org.apache.tika.parser.microsoft.EMFParser", Arrays.asList(metadata.getValues(TikaCoreProperties.TIKA_PARSED_BY_FULL_SET))); } + /** + * 3.x served /tika/text from a BodyContentHandler. 4.x switched to TEXT, which runs over + * the whole XHTML document and emits the <title> as leading character data -- a 4.x + * regression, not a longstanding difference. Needs a real parser: the mock fixtures in + * tika-server-core never put a title in the XHTML, so TEXT and BODY are identical there. + */ + @Test + public void testTextIsBodyOnly() throws Exception { + String text = getStringFromInputStream((InputStream) WebClient + .create(endPoint + TIKA_PATH + "/text") + .put(ClassLoader.getSystemResourceAsStream("test-documents/testHTML.html")) + .getEntity()); + assertContains("Test Indexation Html", text); + assertNotFound("Title : Test Indexation Html", text); + + // the whole-document handler is still reachable explicitly, and still emits the title + String whole = getStringFromInputStream((InputStream) WebClient + .create(endPoint + TIKA_PATH + "/json/text") + .accept("application/json") + .put(ClassLoader.getSystemResourceAsStream("test-documents/testHTML.html")) + .getEntity()); + assertContains("Title : Test Indexation Html", whole); + } + }
