This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4809-stage-4 in repository https://gitbox.apache.org/repos/asf/tika.git
commit 871d2c0db9dfb861c7359c493ee996daa3a619eb Author: tallison <[email protected]> AuthorDate: Sun Aug 9 20:28:19 2026 -0400 TIKA-4809: Remove the writeLimit and throwOnWriteLimitReached headers --- .../core/resource/RecursiveMetadataResource.java | 11 +- .../server/core/resource/ServerHandlerConfig.java | 4 +- .../tika/server/core/resource/TikaResource.java | 87 ++--------- .../server/core/RecursiveMetadataResourceTest.java | 61 -------- .../apache/tika/server/core/TikaResourceTest.java | 30 ---- .../server/core/benchmark/TikaServerBenchmark.java | 1 - .../standard/RecursiveMetadataResourceTest.java | 160 +++++---------------- .../tika/server/standard/TikaResourceTest.java | 59 -------- 8 files changed, 55 insertions(+), 358 deletions(-) diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java index 0f858c909c..762b8ab0ab 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java @@ -17,7 +17,6 @@ package org.apache.tika.server.core.resource; import static org.apache.tika.server.core.resource.TikaResource.fillMetadata; -import static org.apache.tika.server.core.resource.TikaResource.getWriteLimit; import static org.apache.tika.server.core.resource.TikaResource.setupContentHandlerFactory; import static org.apache.tika.server.core.resource.TikaResource.setupContentHandlerFactoryIfNeeded; @@ -74,8 +73,7 @@ public class RecursiveMetadataResource { TikaResource.logRequest(LOG, "/rmeta", metadata); // Set up handler factory in context using shared utility - setupContentHandlerFactory(context, handlerConfig.type().toString(), handlerConfig.writeLimit(), - handlerConfig.throwOnWriteLimitReached()); + setupContentHandlerFactory(context, handlerConfig.type().toString()); // Set up embedded limits if specified if (handlerConfig.maxEmbeddedCount() >= 0) { @@ -96,8 +94,8 @@ public class RecursiveMetadataResource { } else if (httpHeaders.containsKey("maxEmbeddedCount")) { maxEmbeddedCount = Integer.parseInt(httpHeaders.getFirst("maxEmbeddedCount")); } - return new ServerHandlerConfig(BasicContentHandlerFactory.parseHandlerType(handlerTypeName, DEFAULT_HANDLER_TYPE), parseMode, - getWriteLimit(httpHeaders), maxEmbeddedCount, TikaResource.getThrowOnWriteLimitReached(httpHeaders)); + return new ServerHandlerConfig(BasicContentHandlerFactory.parseHandlerType(handlerTypeName, DEFAULT_HANDLER_TYPE), + parseMode, maxEmbeddedCount); } /** @@ -169,8 +167,7 @@ public class RecursiveMetadataResource { private MetadataList parseMetadataWithContext(TikaInputStream tis, Metadata metadata, MultivaluedMap<String, String> httpHeaders, ServerHandlerConfig handlerConfig, ParseContext context) throws Exception { // Set up handler factory in context if not already set using shared utility - setupContentHandlerFactoryIfNeeded(context, handlerConfig.type().toString(), - handlerConfig.writeLimit(), handlerConfig.throwOnWriteLimitReached()); + setupContentHandlerFactoryIfNeeded(context, handlerConfig.type().toString()); // Filtering is done in child process, no need to filter again List<Metadata> metadataList = tikaResource.parseWithPipes(tis, metadata, context, ParseMode.RMETA); diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/ServerHandlerConfig.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/ServerHandlerConfig.java index 53325f4848..5187563d95 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/ServerHandlerConfig.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/ServerHandlerConfig.java @@ -30,8 +30,6 @@ import org.apache.tika.sax.BasicContentHandlerFactory; public record ServerHandlerConfig( BasicContentHandlerFactory.HANDLER_TYPE type, ParseMode parseMode, - int writeLimit, - int maxEmbeddedCount, - boolean throwOnWriteLimitReached + int maxEmbeddedCount ) { } diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java index 5ac7fe1b4b..a6ed9acf7c 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java @@ -407,65 +407,18 @@ public class TikaResource { } } - public static boolean getThrowOnWriteLimitReached(MultivaluedMap<String, String> httpHeaders) { - if (httpHeaders.containsKey("throwOnWriteLimitReached")) { - String val = httpHeaders.getFirst("throwOnWriteLimitReached"); - if ("true".equalsIgnoreCase(val)) { - return true; - } else if ("false".equalsIgnoreCase(val)) { - return false; - } else { - throw new IllegalArgumentException("'throwOnWriteLimitReached' must be either 'true' or 'false'"); - } - } - // Default: throw on write limit reached - return true; - } - - /** - * Parses the writeLimit header value from HTTP headers. - * - * @param httpHeaders the HTTP headers - * @return the write limit value, or -1 if not specified - */ - public static int getWriteLimit(MultivaluedMap<String, String> httpHeaders) { - if (httpHeaders.containsKey("writeLimit")) { - return Integer.parseInt(httpHeaders.getFirst("writeLimit")); - } - return -1; - } - /** - * Sets up the ContentHandlerFactory in the ParseContext based on handler type and HTTP headers. - * This is a shared utility method used by both /tika and /rmeta endpoints. + * Sets up the ContentHandlerFactory in the ParseContext, taking the write limits from + * {@link org.apache.tika.config.OutputLimits} in the context. * * @param context the ParseContext to configure * @param handlerTypeName the handler type name (text, html, xml, ignore), may be null for default - * @param httpHeaders the HTTP headers containing writeLimit and throwOnWriteLimitReached */ - public static void setupContentHandlerFactory(ParseContext context, String handlerTypeName, - MultivaluedMap<String, String> httpHeaders) { - int writeLimit = getWriteLimit(httpHeaders); - boolean throwOnWriteLimitReached = getThrowOnWriteLimitReached(httpHeaders); - setupContentHandlerFactory(context, handlerTypeName, writeLimit, throwOnWriteLimitReached); - } - - /** - * Sets up the ContentHandlerFactory in the ParseContext based on explicit parameters. - * This overload is used when the values have already been parsed (e.g., from ServerHandlerConfig). - * - * @param context the ParseContext to configure - * @param handlerTypeName the handler type name (text, html, xml, ignore), may be null for default - * @param writeLimit the write limit, or -1 for unlimited - * @param throwOnWriteLimitReached whether to throw when write limit is reached - */ - public static void setupContentHandlerFactory(ParseContext context, String handlerTypeName, - int writeLimit, boolean throwOnWriteLimitReached) { + public static void setupContentHandlerFactory(ParseContext context, String handlerTypeName) { BasicContentHandlerFactory.HANDLER_TYPE type = BasicContentHandlerFactory.parseHandlerType( handlerTypeName, DEFAULT_HANDLER_TYPE); - ContentHandlerFactory factory = new BasicContentHandlerFactory(type, writeLimit, - throwOnWriteLimitReached, context); - context.set(ContentHandlerFactory.class, factory); + context.set(ContentHandlerFactory.class, + BasicContentHandlerFactory.newInstance(type, context)); } /** @@ -474,28 +427,10 @@ public class TikaResource { * * @param context the ParseContext to configure * @param handlerTypeName the handler type name - * @param httpHeaders the HTTP headers - */ - public static void setupContentHandlerFactoryIfNeeded(ParseContext context, String handlerTypeName, - MultivaluedMap<String, String> httpHeaders) { - if (context.get(ContentHandlerFactory.class) == null) { - setupContentHandlerFactory(context, handlerTypeName, httpHeaders); - } - } - - /** - * Sets up the ContentHandlerFactory in the ParseContext if not already set. - * This overload is used when the values have already been parsed. - * - * @param context the ParseContext to configure - * @param handlerTypeName the handler type name - * @param writeLimit the write limit, or -1 for unlimited - * @param throwOnWriteLimitReached whether to throw when write limit is reached */ - public static void setupContentHandlerFactoryIfNeeded(ParseContext context, String handlerTypeName, - int writeLimit, boolean throwOnWriteLimitReached) { + public static void setupContentHandlerFactoryIfNeeded(ParseContext context, String handlerTypeName) { if (context.get(ContentHandlerFactory.class) == null) { - setupContentHandlerFactory(context, handlerTypeName, writeLimit, throwOnWriteLimitReached); + setupContentHandlerFactory(context, handlerTypeName); } } @@ -765,7 +700,7 @@ public class TikaResource { String handlerTypeName) throws IOException { fillMetadata(null, metadata, httpHeaders); ParseContext context = createParseContext(); - setupContentHandlerFactory(context, handlerTypeName, httpHeaders); + setupContentHandlerFactory(context, handlerTypeName); return produceRawOutputWithContext(tis, metadata, context, handlerTypeName); } @@ -781,7 +716,7 @@ public class TikaResource { logRequest(LOG, "/tika", metadata); // Ensure content handler factory is set (config may have set it) - setupContentHandlerFactoryIfNeeded(context, handlerTypeName, -1, true); + setupContentHandlerFactoryIfNeeded(context, handlerTypeName); LOG.debug("produceRawOutput: handlerType={}, contentHandlerFactory={}", handlerTypeName, context.get(ContentHandlerFactory.class)); @@ -849,7 +784,7 @@ public class TikaResource { String handlerTypeName) throws IOException { fillMetadata(null, metadata, headers); ParseContext context = createParseContext(); - setupContentHandlerFactory(context, handlerTypeName, headers); + setupContentHandlerFactory(context, handlerTypeName); return produceJsonWithContext(tis, metadata, context, handlerTypeName); } @@ -871,7 +806,7 @@ public class TikaResource { logRequest(LOG, "/tika", metadata); // Ensure content handler factory is set (config may have set it) - setupContentHandlerFactoryIfNeeded(context, handlerTypeName, -1, true); + setupContentHandlerFactoryIfNeeded(context, handlerTypeName); List<Metadata> metadataList; try { diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/RecursiveMetadataResourceTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/RecursiveMetadataResourceTest.java index b90e661979..99907fd910 100644 --- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/RecursiveMetadataResourceTest.java +++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/RecursiveMetadataResourceTest.java @@ -91,66 +91,5 @@ public class RecursiveMetadataResourceTest extends CXFTestBase { Reader reader = new InputStreamReader((InputStream) response.getEntity(), UTF_8); return JsonMetadataList.fromJson(reader).get(0).get(TikaCoreProperties.TIKA_CONTENT); } - /* - @Test - public void testWriteLimitInAll() throws Exception { - //specify your file directory here - Path testDocs = Paths.get("..../tika-parsers/src/test/resources/test-documents"); - for (File f : testDocs.toFile().listFiles()) { - if (f.isDirectory()) { - continue; - } - testWriteLimit(f); - } - } - private void testWriteLimit(File f) throws Exception { - Response response = WebClient.create(endPoint + META_PATH+"/text").accept( - "application/json") - .put(f); - assertEquals(200, response.getStatus()); - Reader reader = new InputStreamReader((InputStream) response.getEntity(), UTF_8); - List<Metadata> metadataList = JsonMetadataList.fromJson(reader); - int totalLen = 0; - StringBuilder sb = new StringBuilder(); - for (Metadata m : metadataList) { - String txt = m.get(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT); - sb.append(txt); - totalLen += (txt == null) ? 0 : txt.length(); - } - String fullText = sb.toString(); - Random r = new Random(); - for (int i = 0; i < 20; i++) { - int writeLimit = r.nextInt(totalLen+100); - response = WebClient.create(endPoint + META_PATH+"/text").accept( - "application/json") - .header("writeLimit", Integer.toString(writeLimit)).put(f); - assertEquals(200, response.getStatus()); - reader = new InputStreamReader((InputStream) response.getEntity(), UTF_8); - List<Metadata> writeLimitMetadataList = JsonMetadataList.fromJson(reader); - int len = 0; - StringBuilder extracted = new StringBuilder(); - for (Metadata m : writeLimitMetadataList) { - String txt = m.get(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT); - len += (txt == null) ? 0 : txt.length(); - extracted.append(txt); - } - if (totalLen > len) { - boolean wlr = false; - for (Metadata m : writeLimitMetadataList) { - if ("true".equals(m.get(AbstractRecursiveParserWrapperHandler.WRITE_LIMIT_REACHED))) { - wlr = true; - } - } - assertTrue(f.getName() + ": writelimit: " + writeLimit + " len: "+len, - len <= writeLimit); - assertEquals(f.getName() +" writeLimit: " + writeLimit + - " : fullLen:" + totalLen + " limitedLen: " +len, - true, wlr); - } else if (len > totalLen) { - fail("len should never be > totalLen "+len + " : "+ totalLen); - } - } - } - */ } diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaResourceTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaResourceTest.java index c0f01e0863..8e89a38980 100644 --- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaResourceTest.java +++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/TikaResourceTest.java @@ -116,24 +116,6 @@ public class TikaResourceTest extends CXFTestBase { assertNotFound("null pointer message", metadata.get(TikaCoreProperties.CONTAINER_EXCEPTION)); } - @Test - public void testJsonWriteLimit() throws Exception { - Response response = WebClient - .create(endPoint + TIKA_PATH + "/json") - .header("writeLimit", "100") - .put(ClassLoader.getSystemResourceAsStream(TEST_HELLO_WORLD_LONG)); - Metadata metadata = JsonMetadata.fromJson(new InputStreamReader(((InputStream) response.getEntity()), StandardCharsets.UTF_8)); - - assertEquals("Nikolai Lobachevsky", metadata.get("author")); - assertEquals("application/mock+xml", metadata.get(Metadata.CONTENT_TYPE)); - assertContains("Hello world", metadata.get(TikaCoreProperties.TIKA_CONTENT)); - assertNotFound("dissolve", metadata.get(TikaCoreProperties.TIKA_CONTENT)); - assertTrue(metadata - .get(TikaCoreProperties.CONTAINER_EXCEPTION) - .startsWith("org.apache.tika.exception.WriteLimitReachedException")); - assertEquals("true", metadata.get(TikaCoreProperties.WRITE_LIMIT_REACHED)); - } - @Test public void testJsonHandlerType() throws Exception { // Default /tika/json uses text handler @@ -161,18 +143,6 @@ public class TikaResourceTest extends CXFTestBase { } /* - @Test - public void testWriteLimitInAll() throws Exception { - //specify your file directory here - Path testDocs = Paths.get("..../tika-parsers/src/test/resources/test-documents"); - for (File f : testDocs.toFile().listFiles()) { - if (f.isDirectory()) { - continue; - } - testWriteLimit(f); - } - } - private void testWriteLimit(File f) throws Exception { Response response = WebClient.create(endPoint + TIKA_PATH + "/text").accept("application/json").put(f); diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/benchmark/TikaServerBenchmark.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/benchmark/TikaServerBenchmark.java index fd968bd8cc..7e76c4f64c 100644 --- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/benchmark/TikaServerBenchmark.java +++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/benchmark/TikaServerBenchmark.java @@ -339,7 +339,6 @@ public class TikaServerBenchmark { .uri(URI.create(baseUrl + "/rmeta")) .header("Content-Type", "application/mock+xml") .header("Accept", "application/json") - .header("writeLimit", "-1") .PUT(HttpRequest.BodyPublishers.ofString(testXml)) .timeout(Duration.ofSeconds(10)) .build(); diff --git a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/RecursiveMetadataResourceTest.java b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/RecursiveMetadataResourceTest.java index 21efbd7300..025c4593c1 100644 --- a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/RecursiveMetadataResourceTest.java +++ b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/RecursiveMetadataResourceTest.java @@ -469,155 +469,73 @@ public class RecursiveMetadataResourceTest extends CXFTestBase { assertContains("plundered our seas", content); } - @Test - @org.junit.jupiter.api.Disabled("maxEmbeddedResources header not yet supported with pipes-based parsing") - public void testEmbeddedResourceLimit() throws Exception { - for (int i : new int[]{0, 1, 5}) { - Response response = WebClient - .create(endPoint + META_PATH) - .accept("application/json") - .header("maxEmbeddedResources", Integer.toString(i)) - .put(ClassLoader.getSystemResourceAsStream(TEST_RECURSIVE_DOC)); - - assertEquals(200, response.getStatus()); - // Check results - Reader reader = new InputStreamReader((InputStream) response.getEntity(), UTF_8); - List<Metadata> metadataList = JsonMetadataList.fromJson(reader); - assertEquals(i + 1, metadataList.size()); - } - } - // TIKA-3227 - TODO: re-enable once maxEmbeddedResources is configurable via JSON // Use maxEmbeddedResources=0 in config to skip embedded documents @Test - public void testWriteLimit() throws Exception { - int writeLimit = 10; + public void testXFA() throws Exception { Response response = WebClient .create(endPoint + META_PATH) .accept("application/json") - .header("writeLimit", Integer.toString(writeLimit)) - .put(ClassLoader.getSystemResourceAsStream(TEST_RECURSIVE_DOC)); + .put(ClassLoader.getSystemResourceAsStream( + "test-documents/testPDF_XFA_govdocs1_258578.pdf")); assertEquals(200, response.getStatus()); - // Check results Reader reader = new InputStreamReader((InputStream) response.getEntity(), UTF_8); List<Metadata> metadataList = JsonMetadataList.fromJson(reader); assertEquals(1, metadataList.size()); - assertEquals("true", metadataList - .get(0) - .get(TikaCoreProperties.WRITE_LIMIT_REACHED)); + Metadata m = metadataList.get(0); + assertEquals("true", m.get(PDF.HAS_XFA)); + assertTrue(m.get(TikaCoreProperties.TIKA_CONTENT).contains("Young Abraham Lincoln")); + } - //now try with a write limit of 500 - writeLimit = 550; - response = WebClient - .create(endPoint + META_PATH) - .accept("application/json") - .header("writeLimit", Integer.toString(writeLimit)) - .put(ClassLoader.getSystemResourceAsStream(TEST_RECURSIVE_DOC)); + /** + * Write limits moved from the `writeLimit` HTTP header to output-limits in the + * config. Truncation semantics themselves are covered by + * BasicContentHandlerFactoryTest; what these two assert is that the server + * actually plumbs a configured limit through to the response. + */ + @Test + public void testWriteLimitFromConfig() throws Exception { + List<Metadata> metadataList = parseWithOutputLimits( + "{\"output-limits\": {\"writeLimit\": 100, \"throwOnWriteLimit\": true}}"); - assertEquals(200, response.getStatus()); - // Check results - reader = new InputStreamReader((InputStream) response.getEntity(), UTF_8); - metadataList = JsonMetadataList.fromJson(reader); - assertEquals(10, metadataList.size()); - // Verify write limit was reached and content was partially extracted - // (order may vary based on ZIP entry iteration) - boolean foundWriteLimitReached = false; - int totalContentLength = 0; + int total = 0; for (Metadata m : metadataList) { - if ("true".equals(m.get(TikaCoreProperties.WRITE_LIMIT_REACHED))) { - foundWriteLimitReached = true; - } String content = m.get(TikaCoreProperties.TIKA_CONTENT); - if (content != null) { - totalContentLength += content.length(); - } + total += (content == null) ? 0 : content.length(); } - assertTrue(foundWriteLimitReached, "Should have reached write limit"); - assertTrue(totalContentLength > 0, "Should have extracted some content"); - + assertTrue(total <= 100, "content should be bounded by the configured writeLimit, was " + total); } @Test - public void testWriteLimitInPDF() throws Exception { - int writeLimit = 10; - Response response = WebClient - .create(endPoint + META_PATH) - .accept("application/json") - .header("writeLimit", Integer.toString(writeLimit)) - .put(ClassLoader.getSystemResourceAsStream("test-documents/testPDFTwoTextBoxes" + ".pdf")); + public void testNoThrowOnWriteLimitFromConfig() throws Exception { + List<Metadata> metadataList = parseWithOutputLimits( + "{\"output-limits\": {\"writeLimit\": 100, \"throwOnWriteLimit\": false}}"); - assertEquals(200, response.getStatus()); - Reader reader = new InputStreamReader((InputStream) response.getEntity(), UTF_8); - List<Metadata> metadataList = JsonMetadataList.fromJson(reader); - Metadata metadata = metadataList.get(0); - assertEquals("true", metadata.get(TikaCoreProperties.WRITE_LIMIT_REACHED)); + boolean limitReached = false; + for (Metadata m : metadataList) { + if ("true".equals(m.get(TikaCoreProperties.WRITE_LIMIT_REACHED))) { + limitReached = true; + } + } + assertTrue(limitReached, "write-limit-reached should be reported when throwOnWriteLimit is false"); } - @Test - public void testXFA() throws Exception { - Response response = WebClient - .create(endPoint + META_PATH) - .accept("application/json") - .put(ClassLoader.getSystemResourceAsStream( - "test-documents/testPDF_XFA_govdocs1_258578.pdf")); - - assertEquals(200, response.getStatus()); - Reader reader = new InputStreamReader((InputStream) response.getEntity(), UTF_8); - List<Metadata> metadataList = JsonMetadataList.fromJson(reader); - assertEquals(1, metadataList.size()); - Metadata m = metadataList.get(0); - assertEquals("true", m.get(PDF.HAS_XFA)); - assertTrue(m.get(TikaCoreProperties.TIKA_CONTENT).contains("Young Abraham Lincoln")); - } + private List<Metadata> parseWithOutputLimits(String configJson) throws Exception { + Attachment fileAtt = new Attachment("file", "application/octet-stream", + ClassLoader.getSystemResourceAsStream(TEST_RECURSIVE_DOC)); + Attachment configAtt = new Attachment("config", "application/json", + new java.io.ByteArrayInputStream(configJson.getBytes(UTF_8))); - @Test - public void testNoThrowOnWriteLimitReached() throws Exception { - int writeLimit = 100; Response response = WebClient - .create(endPoint + META_PATH) + .create(endPoint + META_PATH + "/config") + .type("multipart/form-data") .accept("application/json") - .header("writeLimit", Integer.toString(writeLimit)) - .header("throwOnWriteLimitReached", "false") - .put(ClassLoader.getSystemResourceAsStream(TEST_RECURSIVE_DOC)); + .post(new MultipartBody(Arrays.asList(fileAtt, configAtt))); assertEquals(200, response.getStatus()); - // Check results Reader reader = new InputStreamReader((InputStream) response.getEntity(), UTF_8); - List<Metadata> metadataList = JsonMetadataList.fromJson(reader); - assertEquals(12, metadataList.size()); - assertEquals("true", metadataList - .get(0) - .get(TikaCoreProperties.WRITE_LIMIT_REACHED)); - - //now try with a write limit of 550 - writeLimit = 550; - response = WebClient - .create(endPoint + META_PATH) - .accept("application/json") - .header("writeLimit", Integer.toString(writeLimit)) - .header("throwOnWriteLimitReached", "false") - .put(ClassLoader.getSystemResourceAsStream(TEST_RECURSIVE_DOC)); - - assertEquals(200, response.getStatus()); - // Check results - reader = new InputStreamReader((InputStream) response.getEntity(), UTF_8); - metadataList = JsonMetadataList.fromJson(reader); - assertEquals(12, metadataList.size()); - assertEquals("true", metadataList - .get(0) - .get(TikaCoreProperties.WRITE_LIMIT_REACHED)); - // Verify content was partially extracted (order may vary based on ZIP entry iteration) - int totalContentLength = 0; - for (Metadata m : metadataList) { - String content = m.get(TikaCoreProperties.TIKA_CONTENT); - if (content != null) { - totalContentLength += content.length(); - } - } - assertTrue(totalContentLength > 0, "Should have extracted some content"); - + return JsonMetadataList.fromJson(reader); } - } diff --git a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaResourceTest.java b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaResourceTest.java index fce1711635..91df02fb2c 100644 --- a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaResourceTest.java +++ b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaResourceTest.java @@ -16,7 +16,6 @@ */ package org.apache.tika.server.standard; -import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.cxf.helpers.HttpHeaderHelper.CONTENT_ENCODING; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -26,7 +25,6 @@ import static org.junit.jupiter.api.Assumptions.assumeTrue; import java.io.InputStream; import java.io.InputStreamReader; -import java.io.Reader; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -95,14 +93,6 @@ public class TikaResourceTest extends CXFTestBase { return getClass().getResourceAsStream("/configs/tika-config-for-server-tests.json"); } - @Test - public void testHelloWorld() throws Exception { - Response response = WebClient - .create(endPoint + TIKA_PATH) - .get(); - assertEquals(TikaResource.GREETING, getStringFromInputStream((InputStream) response.getEntity())); - } - @Test public void testSimpleWord() throws Exception { Response response = WebClient @@ -655,53 +645,4 @@ public class TikaResourceTest extends CXFTestBase { TikaTest.assertContains("org.apache.tika.parser.microsoft.EMFParser", Arrays.asList(metadata.getValues(TikaCoreProperties.TIKA_PARSED_BY_FULL_SET))); } - @Test - public void testJsonWriteLimitEmbedded() throws Exception { - Response response = WebClient - .create(endPoint + TIKA_PATH + "/json/html") - .header("writeLimit", "500") - .put(ClassLoader.getSystemResourceAsStream(TEST_RECURSIVE_DOC)); - Metadata metadata = JsonMetadata.fromJson(new InputStreamReader(((InputStream) response.getEntity()), StandardCharsets.UTF_8)); - assertContains("embed2a.txt", metadata.get(TikaCoreProperties.TIKA_CONTENT)); - assertContains("When in the Course", metadata.get(TikaCoreProperties.TIKA_CONTENT)); - assertNotFound("declare the causes", metadata.get(TikaCoreProperties.TIKA_CONTENT)); - assertEquals("Microsoft Office Word", metadata.get(OfficeOpenXMLExtended.APPLICATION)); - assertTrue(metadata - .get(TikaCoreProperties.CONTAINER_EXCEPTION) - .startsWith("org.apache.tika.exception.WriteLimitReachedException")); - assertNotFound("embed4.txt", metadata.get(TikaCoreProperties.TIKA_CONTENT)); - } - - @Test - @org.junit.jupiter.api.Disabled("throwOnWriteLimitReached header not yet supported with pipes-based parsing") - public void testJsonNoThrowWriteLimitEmbedded() throws Exception { - Response response = WebClient - .create(endPoint + TIKA_PATH + "/json/html") - .header("writeLimit", "500") - .header("throwOnWriteLimitReached", "false") - .put(ClassLoader.getSystemResourceAsStream(TEST_RECURSIVE_DOC)); - Metadata metadata = JsonMetadata.fromJson(new InputStreamReader(((InputStream) response.getEntity()), StandardCharsets.UTF_8)); - String txt = metadata.get(TikaCoreProperties.TIKA_CONTENT); - assertContains("embed2a.txt", txt); - assertContains("When in the Course", txt); - assertNotFound("declare the causes", txt); - assertEquals("Microsoft Office Word", metadata.get(OfficeOpenXMLExtended.APPLICATION)); - assertEquals("true", metadata.get(TikaCoreProperties.WRITE_LIMIT_REACHED)); - assertContains("<div class=\"embedded\" id=\"embed4.txt", metadata.get(TikaCoreProperties.TIKA_CONTENT)); - } - - @Test - public void testWriteLimitInPDF() throws Exception { - int writeLimit = 10; - Response response = WebClient - .create(endPoint + TIKA_PATH + "/json") - .header("writeLimit", Integer.toString(writeLimit)) - .put(ClassLoader.getSystemResourceAsStream("test-documents/testPDFTwoTextBoxes.pdf")); - - assertEquals(200, response.getStatus()); - Reader reader = new InputStreamReader((InputStream) response.getEntity(), UTF_8); - Metadata metadata = JsonMetadata.fromJson(reader); - assertEquals("true", metadata.get(TikaCoreProperties.WRITE_LIMIT_REACHED)); - - } }
