This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4848 in repository https://gitbox.apache.org/repos/asf/tika.git
commit 8531dd6ca6556549e70728fe17d5a6975532caab Author: tallison <[email protected]> AuthorDate: Thu Aug 27 16:21:52 2026 -0400 TIKA-4848: add exception-reporting parse-context config, step 1 --- CHANGES.txt | 10 ++ .../ROOT/pages/advanced/setting-limits.adoc | 47 ++++++ .../org/apache/tika/config/ExceptionReporting.java | 136 ++++++++++++++++ .../tika/extractor/EmbeddedDocumentUtil.java | 27 +++- .../org/apache/tika/parser/CompositeParser.java | 2 +- .../apache/tika/parser/RecursiveParserWrapper.java | 8 +- .../parser/multiple/AbstractMultipleParser.java | 4 +- .../java/org/apache/tika/utils/ExceptionUtils.java | 179 +++++++++++++++++---- .../java/org/apache/tika/utils/ParserUtils.java | 16 +- .../tika/config/ExceptionReportingParseTest.java | 95 +++++++++++ .../org/apache/tika/utils/ExceptionUtilsTest.java | 164 +++++++++++++++++++ .../tika/eval/core/util/EvalExceptionUtils.java | 8 +- .../eval/core/util/EvalExceptionUtilsTest.java | 45 ++++++ .../java/org/apache/tika/parser/pkg/ZipParser.java | 2 +- .../WireRestrictedFetchEmitTupleTest.java | 9 ++ .../tika/serialization/ComponentNameResolver.java | 22 ++- .../apache/tika/config/ExceptionReportingTest.java | 45 ++++++ .../WireRestrictedParseContextTest.java | 25 +++ .../configs/exception-reporting-test.json | 8 + 19 files changed, 800 insertions(+), 52 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 5ae3270114..e4e41ad655 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,15 @@ Release 4.1.0 - unreleased + * New exception-reporting parse-context config controls how much of an + exception is written to tk:exception:* metadata for the container and + embedded documents alike: FULL (default), MESSAGE_REDACTED (stack trace + without messages) or REDACTED (class names only), plus a maxLength. + Config-only; rejected in per-request parse-context. Parsers should + format exceptions via ExceptionUtils.format(Throwable, ParseContext); + ExceptionUtils.getStackTrace/getFilteredStackTrace and the ParseContext-less + EmbeddedDocumentUtil.recordException* / ParserUtils.recordParserFailure + overloads are deprecated (TIKA-4848). + * tika-pipes: the cache memory budget (how much rewindable content a forked worker keeps in memory before spilling to disk; new since 4.0.0, which had no budget at all) defaults to a quarter of the fork's heap, so raising diff --git a/docs/modules/ROOT/pages/advanced/setting-limits.adoc b/docs/modules/ROOT/pages/advanced/setting-limits.adoc index 2d933a3a6b..d1f4fe0848 100644 --- a/docs/modules/ROOT/pages/advanced/setting-limits.adoc +++ b/docs/modules/ROOT/pages/advanced/setting-limits.adoc @@ -47,6 +47,10 @@ throughout the parse. |`standard-metadata-limiter-factory` |`StandardMetadataLimiterFactory` |Metadata size, per-field size, key size, values per field, field allow/deny lists. + +|`exception-reporting` +|`ExceptionReporting` +|How much of an exception (stack trace, message, class name) is reported, and its length. |=== == Configuring limits @@ -346,9 +350,52 @@ if ("true".equals(metadata.get(TikaCoreProperties.TRUNCATED_METADATA))) { } ---- +== Exception reporting + +`ExceptionReporting` controls how much detail Tika reports when a parse fails: +the `tk:exception:container-exception`, `tk:exception:embedded-exception`, +`tk:exception:warn` and `tk:exception:embedded-stream` metadata values. Stack +traces and exception messages can carry file paths, hostnames and fragments of +the document; the metadata limiter deliberately never truncates them (see +<<Always-included fields>>), so this is the only bound on them. + +[cols="2,1,3"] +|=== +|Setting |Default |Description + +|`level` +|`FULL` +|`FULL`: the complete stack trace. `MESSAGE_REDACTED`: the complete stack +trace with every exception message removed (class names, frames and the cause +chain are kept). `REDACTED`: exception class names only, no frames. + +|`maxLength` +|-1 (unlimited) +|Maximum characters in the formatted exception; longer values end with +`...[truncated]`. +|=== + +[source,json] +---- +"exception-reporting": { + "level": "MESSAGE_REDACTED", + "maxLength": 10000 +} +---- + +The same policy applies to the container and to every embedded document. It is +loaded from the config only and is rejected in a per-request `parse-context` +(tika-server `/rmeta/config` and friends, `/pipes`, `/async`), so a caller cannot +turn redaction back off. + +Programmatically, parsers and callers should format exceptions through +`ExceptionUtils.format(Throwable, ParseContext)`. + == Recommendations . Set limits whenever the content is untrusted. +. Set `exception-reporting` to `MESSAGE_REDACTED` with a finite `maxLength` + when the caller is not the operator. . Use `includeFields` to capture only the metadata you need. . Check `tk:warn:truncated-metadata` rather than guessing. . Combine with process isolation — limits protect against memory blowups, diff --git a/tika-core/src/main/java/org/apache/tika/config/ExceptionReporting.java b/tika-core/src/main/java/org/apache/tika/config/ExceptionReporting.java new file mode 100644 index 0000000000..771162dd6c --- /dev/null +++ b/tika-core/src/main/java/org/apache/tika/config/ExceptionReporting.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.config; + +import java.io.Serializable; +import java.util.Objects; + +import org.apache.tika.annotation.TikaComponent; +import org.apache.tika.parser.ParseContext; + +/** + * How much detail Tika reports when an exception is turned into a string: the + * {@code tk:exception:*} metadata values, tika-server error bodies and pipes result messages. + * <p> + * Stack traces and exception messages can carry file paths, hostnames and fragments of the + * document being parsed. Operators who expose Tika to untrusted callers can reduce that with + * {@link Level} and bound the size with {@code maxLength}. + * <ul> + * <li>{@link Level#FULL} - the complete stack trace (default)</li> + * <li>{@link Level#MESSAGE_REDACTED} - the complete stack trace with every exception message + * removed; class names, frames and the cause chain are kept</li> + * <li>{@link Level#REDACTED} - exception class names only (the cause chain, no frames, + * no messages)</li> + * </ul> + * {@code maxLength} truncates the formatted string to that many characters (-1 = unlimited). + * <p> + * Loaded from the {@code parse-context} section of the config and deliberately not settable + * per request: a caller could otherwise turn redaction back off. + * <pre> + * { + * "parse-context": { + * "exception-reporting": { + * "level": "MESSAGE_REDACTED", + * "maxLength": 10000 + * } + * } + * } + * </pre> + * + * @since Apache Tika 4.1 + */ +@TikaComponent(name = "exception-reporting", spi = false) +public class ExceptionReporting implements Serializable { + + private static final long serialVersionUID = 1L; + + public enum Level { + REDACTED, MESSAGE_REDACTED, FULL + } + + public static final int UNLIMITED = -1; + + public static final ExceptionReporting DEFAULT = new ExceptionReporting(); + + private Level level = Level.FULL; + private int maxLength = UNLIMITED; + + public ExceptionReporting() { + } + + public ExceptionReporting(Level level, int maxLength) { + setLevel(level); + setMaxLength(maxLength); + } + + public Level getLevel() { + return level; + } + + public void setLevel(Level level) { + this.level = Objects.requireNonNull(level, "level"); + } + + public int getMaxLength() { + return maxLength; + } + + /** + * @param maxLength maximum characters in the formatted exception, or -1 for unlimited + */ + public void setMaxLength(int maxLength) { + if (maxLength < UNLIMITED || maxLength == 0) { + throw new IllegalArgumentException( + "maxLength must be positive or -1 for unlimited, was " + maxLength); + } + this.maxLength = maxLength; + } + + /** + * @return the ExceptionReporting from the context, or {@link #DEFAULT} if the context is + * null or has none + */ + public static ExceptionReporting get(ParseContext context) { + if (context == null) { + return DEFAULT; + } + ExceptionReporting reporting = context.get(ExceptionReporting.class); + return reporting != null ? reporting : DEFAULT; + } + + @Override + public String toString() { + return "ExceptionReporting{level=" + level + ", maxLength=" + maxLength + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ExceptionReporting that = (ExceptionReporting) o; + return maxLength == that.maxLength && level == that.level; + } + + @Override + public int hashCode() { + return 31 * level.hashCode() + maxLength; + } +} diff --git a/tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentUtil.java b/tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentUtil.java index d05e91187b..24d8074cba 100644 --- a/tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentUtil.java +++ b/tika-core/src/main/java/org/apache/tika/extractor/EmbeddedDocumentUtil.java @@ -231,14 +231,33 @@ public class EmbeddedDocumentUtil { metadata.set(TikaCoreProperties.RESOURCE_NAME_EXTENSION_INFERRED, true); } + public static void recordException(Throwable t, Metadata m, ParseContext context) { + m.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, + ExceptionUtils.format(ExceptionUtils.unwrapTikaException(t), context)); + } + + /** + * @deprecated use {@link #recordException(Throwable, Metadata, ParseContext)} so the + * configured {@link org.apache.tika.config.ExceptionReporting} applies + */ + @Deprecated public static void recordException(Throwable t, Metadata m) { - String ex = ExceptionUtils.getFilteredStackTrace(t); - m.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING, ex); + recordException(t, m, null); } + public static void recordEmbeddedStreamException(Throwable t, Metadata m, + ParseContext context) { + m.add(TikaCoreProperties.TIKA_META_EXCEPTION_EMBEDDED_STREAM, + ExceptionUtils.format(ExceptionUtils.unwrapTikaException(t), context)); + } + + /** + * @deprecated use {@link #recordEmbeddedStreamException(Throwable, Metadata, ParseContext)} + * so the configured {@link org.apache.tika.config.ExceptionReporting} applies + */ + @Deprecated public static void recordEmbeddedStreamException(Throwable t, Metadata m) { - String ex = ExceptionUtils.getFilteredStackTrace(t); - m.add(TikaCoreProperties.TIKA_META_EXCEPTION_EMBEDDED_STREAM, ex); + recordEmbeddedStreamException(t, m, null); } /** diff --git a/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java b/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java index 9bbfe6e47b..fe198a75b5 100644 --- a/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java +++ b/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java @@ -341,7 +341,7 @@ public class CompositeParser implements Parser { return; } for (Exception e : record.getExceptions()) { - metadata.add(TikaCoreProperties.EMBEDDED_EXCEPTION, ExceptionUtils.getStackTrace(e)); + metadata.add(TikaCoreProperties.EMBEDDED_EXCEPTION, ExceptionUtils.format(e, context)); } for (String msg : record.getWarnings()) { metadata.add(TikaCoreProperties.EMBEDDED_WARNING, msg); diff --git a/tika-core/src/main/java/org/apache/tika/parser/RecursiveParserWrapper.java b/tika-core/src/main/java/org/apache/tika/parser/RecursiveParserWrapper.java index 38b93e7949..e522382132 100644 --- a/tika-core/src/main/java/org/apache/tika/parser/RecursiveParserWrapper.java +++ b/tika-core/src/main/java/org/apache/tika/parser/RecursiveParserWrapper.java @@ -170,8 +170,8 @@ public class RecursiveParserWrapper extends ParserDecorator { if (WriteLimitReachedException.isWriteLimitReached(e)) { metadata.set(TikaCoreProperties.WRITE_LIMIT_REACHED, "true"); } else { - String stackTrace = ExceptionUtils.getFilteredStackTrace(e); - metadata.add(TikaCoreProperties.CONTAINER_EXCEPTION, stackTrace); + metadata.add(TikaCoreProperties.CONTAINER_EXCEPTION, + ExceptionUtils.format(ExceptionUtils.unwrapTikaException(e), context)); throw e; } } finally { @@ -272,7 +272,7 @@ public class RecursiveParserWrapper extends ParserDecorator { throw e; } else { if (catchEmbeddedExceptions) { - ParserUtils.recordParserFailure(this, e, metadata); + ParserUtils.recordParserFailure(this, e, metadata, context); } else { throw e; } @@ -287,7 +287,7 @@ public class RecursiveParserWrapper extends ParserDecorator { e instanceof ZeroByteFileException) { //do nothing } else if (catchEmbeddedExceptions) { - ParserUtils.recordParserFailure(this, e, metadata); + ParserUtils.recordParserFailure(this, e, metadata, context); } else { throw e; } diff --git a/tika-core/src/main/java/org/apache/tika/parser/multiple/AbstractMultipleParser.java b/tika-core/src/main/java/org/apache/tika/parser/multiple/AbstractMultipleParser.java index ee7c55210b..311dfef017 100644 --- a/tika-core/src/main/java/org/apache/tika/parser/multiple/AbstractMultipleParser.java +++ b/tika-core/src/main/java/org/apache/tika/parser/multiple/AbstractMultipleParser.java @@ -277,8 +277,8 @@ public abstract class AbstractMultipleParser implements Parser { p.parse(tis, handler, metadata, context); } catch (Exception e) { // Record the failure such that it can't get lost / overwritten - recordParserFailure(p, e, originalMetadata); - recordParserFailure(p, e, metadata); + recordParserFailure(p, e, originalMetadata, context); + recordParserFailure(p, e, metadata, context); failure = e; } diff --git a/tika-core/src/main/java/org/apache/tika/utils/ExceptionUtils.java b/tika-core/src/main/java/org/apache/tika/utils/ExceptionUtils.java index 88854aa148..8e2e539223 100644 --- a/tika-core/src/main/java/org/apache/tika/utils/ExceptionUtils.java +++ b/tika-core/src/main/java/org/apache/tika/utils/ExceptionUtils.java @@ -17,53 +17,170 @@ package org.apache.tika.utils; -import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; -import java.io.Writer; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Set; +import org.apache.tika.config.ExceptionReporting; import org.apache.tika.exception.TikaException; +import org.apache.tika.parser.ParseContext; +/** + * Turns a {@link Throwable} into the string Tika reports to callers, under the + * {@link ExceptionReporting} policy in effect. Every place Tika records or emits an exception + * as text should go through {@link #format(Throwable, ParseContext)} so that one config + * setting governs all channels. + * <p> + * NOTE: If your stacktraces are truncated, make sure to start your jvm + * with: -XX:-OmitStackTraceInFastThrow + */ public class ExceptionUtils { + private static final int MAX_CAUSE_DEPTH = 64; + private static final String TRUNCATED = "...[truncated]"; + /** - * Simple util to get stack trace. - * <p> - * This will unwrap a TikaException and return the cause if not null - * <p> - * NOTE: If your stacktraces are truncated, make sure to start your jvm - * with: -XX:-OmitStackTraceInFastThrow - * - * @param t throwable - * @return - * @throws IOException + * Formats {@code t} under the {@link ExceptionReporting} found in {@code context} + * (or {@link ExceptionReporting#DEFAULT} if the context is null or has none). */ - public static String getFilteredStackTrace(Throwable t) { - Throwable cause = t; - if ((t.getClass().equals(TikaException.class)) && t.getCause() != null) { - cause = t.getCause(); + public static String format(Throwable t, ParseContext context) { + return format(t, ExceptionReporting.get(context)); + } + + /** + * Formats {@code t} under {@code reporting}; a null policy means + * {@link ExceptionReporting#DEFAULT}. For channels that have no ParseContext, such as + * server error responses and pipes crash messages. + */ + public static String format(Throwable t, ExceptionReporting reporting) { + if (reporting == null) { + reporting = ExceptionReporting.DEFAULT; + } + String s; + switch (reporting.getLevel()) { + case REDACTED: + s = redacted(t, false); + break; + case MESSAGE_REDACTED: + s = redacted(t, true); + break; + default: + s = full(t); } - return getStackTrace(cause); + return truncate(s, reporting.getMaxLength()); } /** - * Get the full stacktrace as a string - * - * @param t - * @return + * A bare {@link TikaException} wrapper adds nothing to a report; returns its cause + * instead. Subclasses and unwrapped throwables are returned as is. */ + public static Throwable unwrapTikaException(Throwable t) { + if (t.getClass().equals(TikaException.class) && t.getCause() != null) { + return t.getCause(); + } + return t; + } + + /** + * @deprecated use {@link #format(Throwable, ParseContext)} on + * {@link #unwrapTikaException(Throwable)} so the configured policy applies + */ + @Deprecated + public static String getFilteredStackTrace(Throwable t) { + return format(unwrapTikaException(t), ExceptionReporting.DEFAULT); + } + + /** + * @deprecated use {@link #format(Throwable, ParseContext)} so the configured policy applies + */ + @Deprecated public static String getStackTrace(Throwable t) { - Writer result = new StringWriter(); - PrintWriter writer = new PrintWriter(result); - t.printStackTrace(writer); - try { - writer.flush(); - result.flush(); - writer.close(); - result.close(); - } catch (IOException e) { - //swallow + return format(t, ExceptionReporting.DEFAULT); + } + + private static String full(Throwable t) { + StringWriter result = new StringWriter(); + try (PrintWriter writer = new PrintWriter(result)) { + t.printStackTrace(writer); } return result.toString(); } + + private static String truncate(String s, int maxLength) { + if (maxLength < 0 || s.length() <= maxLength) { + return s; + } + int cut = maxLength; + if (Character.isHighSurrogate(s.charAt(cut - 1))) { + cut--; + } + return s.substring(0, cut) + TRUNCATED; + } + + /** + * Mirrors {@link Throwable#printStackTrace} (frame elision, suppressed, causes, cycle + * guard) but never calls {@code toString()}/{@code getMessage()}; with {@code frames} + * false only the class-name chain is emitted. + */ + private static String redacted(Throwable t, boolean frames) { + StringBuilder sb = new StringBuilder(); + Set<Throwable> seen = Collections.newSetFromMap(new IdentityHashMap<>()); + seen.add(t); + sb.append(t.getClass().getName()).append('\n'); + StackTraceElement[] trace = t.getStackTrace(); + if (frames) { + for (StackTraceElement e : trace) { + sb.append("\tat ").append(e).append('\n'); + } + } + for (Throwable s : t.getSuppressed()) { + enclosed(s, trace, "Suppressed: ", "\t", seen, sb, frames, 1); + } + Throwable cause = t.getCause(); + if (cause != null) { + enclosed(cause, trace, "Caused by: ", "", seen, sb, frames, 1); + } + return sb.toString(); + } + + private static void enclosed(Throwable t, StackTraceElement[] enclosingTrace, String caption, + String prefix, Set<Throwable> seen, StringBuilder sb, + boolean frames, int depth) { + if (seen.contains(t)) { + sb.append(prefix).append(caption).append("[CIRCULAR REFERENCE: ") + .append(t.getClass().getName()).append("]\n"); + return; + } + seen.add(t); + if (depth > MAX_CAUSE_DEPTH) { + sb.append(prefix).append("... cause chain truncated\n"); + return; + } + sb.append(prefix).append(caption).append(t.getClass().getName()).append('\n'); + StackTraceElement[] trace = t.getStackTrace(); + if (frames) { + int m = trace.length - 1; + int n = enclosingTrace.length - 1; + while (m >= 0 && n >= 0 && trace[m].equals(enclosingTrace[n])) { + m--; + n--; + } + int framesInCommon = trace.length - 1 - m; + for (int i = 0; i <= m; i++) { + sb.append(prefix).append("\tat ").append(trace[i]).append('\n'); + } + if (framesInCommon != 0) { + sb.append(prefix).append("\t... ").append(framesInCommon).append(" more\n"); + } + } + for (Throwable s : t.getSuppressed()) { + enclosed(s, trace, "Suppressed: ", prefix + "\t", seen, sb, frames, depth + 1); + } + Throwable cause = t.getCause(); + if (cause != null) { + enclosed(cause, trace, "Caused by: ", prefix, seen, sb, frames, depth + 1); + } + } } diff --git a/tika-core/src/main/java/org/apache/tika/utils/ParserUtils.java b/tika-core/src/main/java/org/apache/tika/utils/ParserUtils.java index 950048c1b8..02a53ca622 100644 --- a/tika-core/src/main/java/org/apache/tika/utils/ParserUtils.java +++ b/tika-core/src/main/java/org/apache/tika/utils/ParserUtils.java @@ -22,6 +22,7 @@ import java.util.Arrays; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.parser.ParseContext; import org.apache.tika.parser.Parser; import org.apache.tika.parser.ParserDecorator; @@ -82,9 +83,18 @@ public class ParserUtils { * {@link Exception} wasn't immediately thrown (eg when several different * Parsers are used) */ - public static void recordParserFailure(Parser parser, Throwable failure, Metadata metadata) { - String trace = ExceptionUtils.getStackTrace(failure); - metadata.add(EMBEDDED_EXCEPTION, trace); + public static void recordParserFailure(Parser parser, Throwable failure, Metadata metadata, + ParseContext context) { + metadata.add(EMBEDDED_EXCEPTION, ExceptionUtils.format(failure, context)); metadata.add(TikaCoreProperties.EMBEDDED_PARSER, getParserClassname(parser)); } + + /** + * @deprecated use {@link #recordParserFailure(Parser, Throwable, Metadata, ParseContext)} + * so the configured {@link org.apache.tika.config.ExceptionReporting} applies + */ + @Deprecated + public static void recordParserFailure(Parser parser, Throwable failure, Metadata metadata) { + recordParserFailure(parser, failure, metadata, null); + } } diff --git a/tika-core/src/test/java/org/apache/tika/config/ExceptionReportingParseTest.java b/tika-core/src/test/java/org/apache/tika/config/ExceptionReportingParseTest.java new file mode 100644 index 0000000000..60b127fa60 --- /dev/null +++ b/tika-core/src/test/java/org/apache/tika/config/ExceptionReportingParseTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.config.ExceptionReporting.Level; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.parser.ParseContext; + +/** + * The configured level must govern both the container and the embedded exception, in both + * RMETA (RecursiveParserWrapper) and CONCATENATE (CompositeParser roll-up) modes. + */ +public class ExceptionReportingParseTest extends TikaTest { + + private static final String MESSAGE = "another null pointer exception"; + private static final String CLASS = "java.lang.NullPointerException"; + + private static ParseContext context(Level level, int maxLength) { + ParseContext context = new ParseContext(); + context.set(ExceptionReporting.class, new ExceptionReporting(level, maxLength)); + return context; + } + + private static void assertLevel(Level level, String trace) { + assertNotNull(trace); + assertTrue(trace.contains(CLASS), trace); + assertEquals(level == Level.FULL, trace.contains(MESSAGE), level + ":\n" + trace); + assertEquals(level != Level.REDACTED, trace.contains("\tat "), level + ":\n" + trace); + } + + @Test + public void rmetaContainer() throws Exception { + for (Level level : Level.values()) { + List<Metadata> list = + getRecursiveMetadata("embedded_then_npe.xml", context(level, -1), true); + assertLevel(level, list.get(0).get(TikaCoreProperties.CONTAINER_EXCEPTION)); + } + } + + @Test + public void rmetaEmbedded() throws Exception { + for (Level level : Level.values()) { + List<Metadata> list = + getRecursiveMetadata("embedded_with_npe.xml", context(level, -1), true); + assertEquals(2, list.size()); + assertLevel(level, list.get(1).get(TikaCoreProperties.EMBEDDED_EXCEPTION)); + } + } + + @Test + public void concatenateEmbedded() throws Exception { + for (Level level : Level.values()) { + Metadata metadata = getXML("embedded_with_npe.xml", context(level, -1)).metadata; + String[] traces = metadata.getValues(TikaCoreProperties.EMBEDDED_EXCEPTION); + assertEquals(1, traces.length); + assertLevel(level, traces[0]); + } + } + + @Test + public void maxLengthBounds() throws Exception { + for (Level level : Level.values()) { + List<Metadata> list = + getRecursiveMetadata("embedded_with_npe.xml", context(level, 40), true); + String trace = list.get(1).get(TikaCoreProperties.EMBEDDED_EXCEPTION); + assertTrue(trace.endsWith("...[truncated]"), trace); + assertFalse(trace.contains(MESSAGE)); + } + } +} diff --git a/tika-core/src/test/java/org/apache/tika/utils/ExceptionUtilsTest.java b/tika-core/src/test/java/org/apache/tika/utils/ExceptionUtilsTest.java new file mode 100644 index 0000000000..c29f06708b --- /dev/null +++ b/tika-core/src/test/java/org/apache/tika/utils/ExceptionUtilsTest.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.config.ExceptionReporting; +import org.apache.tika.config.ExceptionReporting.Level; +import org.apache.tika.exception.TikaException; +import org.apache.tika.parser.ParseContext; + +public class ExceptionUtilsTest { + + private static final String[] MESSAGES = {"outer secret", "middle secret", "inner secret", + "suppressed secret"}; + + private static Throwable chain() { + IllegalStateException inner = new IllegalStateException(MESSAGES[2]); + IOException middle = new IOException(MESSAGES[1], inner); + TikaException outer = new TikaException(MESSAGES[0], middle); + outer.addSuppressed(new RuntimeException(MESSAGES[3])); + return outer; + } + + private static String format(Throwable t, Level level) { + return ExceptionUtils.format(t, new ExceptionReporting(level, ExceptionReporting.UNLIMITED)); + } + + private static void assertNoMessages(String s) { + for (String m : MESSAGES) { + assertFalse(s.contains(m), "must not contain '" + m + "':\n" + s); + } + } + + @Test + public void fullMatchesPrintStackTrace() { + Throwable t = chain(); + String full = format(t, Level.FULL); + for (String m : MESSAGES) { + assertTrue(full.contains(m)); + } + assertTrue(full.contains("Caused by: java.io.IOException: " + MESSAGES[1])); + } + + @Test + public void messageRedactedEqualsFullMinusMessages() { + Throwable t = chain(); + String full = format(t, Level.FULL); + String redacted = format(t, Level.MESSAGE_REDACTED); + assertNoMessages(redacted); + assertTrue(redacted.contains("\tat ")); + assertTrue(redacted.contains("Caused by: java.io.IOException\n")); + assertTrue(redacted.contains("\tSuppressed: java.lang.RuntimeException\n")); + assertTrue(redacted.contains(" more\n"), "common-frame elision kept"); + // Strip ": message" from every header line of FULL; the rest must be identical. + String expected = full.replaceAll("(?m)^((?:\\t*Suppressed: |Caused by: )?[\\w.$]+Exception): .*$", "$1"); + assertEquals(expected, redacted); + } + + @Test + public void redactedIsClassChainOnly() { + String s = format(chain(), Level.REDACTED); + assertNoMessages(s); + assertFalse(s.contains("\tat ")); + assertEquals("org.apache.tika.exception.TikaException\n" + + "\tSuppressed: java.lang.RuntimeException\n" + + "Caused by: java.io.IOException\n" + + "Caused by: java.lang.IllegalStateException\n", s); + } + + @Test + public void cyclicCauseTerminates() throws Exception { + RuntimeException a = new RuntimeException("a"); + RuntimeException b = new RuntimeException("b", a); + a.initCause(b); + for (Level level : Level.values()) { + String s = format(a, level); + assertTrue(s.contains("CIRCULAR REFERENCE"), level + ":\n" + s); + } + } + + @Test + public void deepChainCapped() { + Throwable t = new RuntimeException("leaf"); + for (int i = 0; i < 200; i++) { + t = new RuntimeException("level " + i, t); + } + String s = format(t, Level.REDACTED); + assertTrue(s.contains("cause chain truncated")); + assertTrue(s.split("\n").length < 100); + } + + @Test + public void maxLengthTruncates() { + Throwable t = chain(); + String s = ExceptionUtils.format(t, new ExceptionReporting(Level.FULL, 50)); + assertTrue(s.startsWith(format(t, Level.FULL).substring(0, 50))); + assertTrue(s.endsWith("...[truncated]")); + assertEquals(50 + "...[truncated]".length(), s.length()); + } + + @Test + public void maxLengthDoesNotSplitSurrogatePair() { + // Cut lands between the two UTF-16 units of the astral char; must back off by one. + Throwable t = new RuntimeException("x😀yyyyyyyy"); + int cut = "java.lang.RuntimeException: x".length() + 1; + String s = ExceptionUtils.format(t, new ExceptionReporting(Level.FULL, cut)); + assertTrue(s.endsWith("x...[truncated]"), s); + } + + @Test + public void nullContextAndPolicyAreFull() { + Throwable t = chain(); + String full = format(t, Level.FULL); + assertEquals(full, ExceptionUtils.format(t, (ParseContext) null)); + assertEquals(full, ExceptionUtils.format(t, (ExceptionReporting) null)); + assertEquals(full, ExceptionUtils.format(t, new ParseContext())); + } + + @Test + public void contextPolicyApplies() { + ParseContext context = new ParseContext(); + context.set(ExceptionReporting.class, new ExceptionReporting(Level.REDACTED, -1)); + assertNoMessages(ExceptionUtils.format(chain(), context)); + } + + @Test + public void unwrapTikaException() { + Throwable t = chain(); + assertEquals(t.getCause(), ExceptionUtils.unwrapTikaException(t)); + Throwable sub = new org.apache.tika.exception.EncryptedDocumentException(t); + assertEquals(sub, ExceptionUtils.unwrapTikaException(sub)); + Throwable bare = new TikaException("no cause"); + assertEquals(bare, ExceptionUtils.unwrapTikaException(bare)); + } + + @Test + public void invalidMaxLength() { + assertThrows(IllegalArgumentException.class, () -> new ExceptionReporting(Level.FULL, 0)); + assertThrows(IllegalArgumentException.class, () -> new ExceptionReporting(Level.FULL, -2)); + assertThrows(NullPointerException.class, () -> new ExceptionReporting(null, -1)); + } +} diff --git a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/util/EvalExceptionUtils.java b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/util/EvalExceptionUtils.java index fd9ff1a6fb..488e8e668b 100644 --- a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/util/EvalExceptionUtils.java +++ b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/util/EvalExceptionUtils.java @@ -29,8 +29,10 @@ public class EvalExceptionUtils { Pattern.compile("(Caused by: [^:]+):[^\\r\\n]+"); //strips the exception message off the first line, so that the same cause with - //differing runtime detail collapses to one entry - private final static Pattern MSG_PATTERN = Pattern.compile(":[^\\r\\n]+"); + //differing runtime detail collapses to one entry. Anchored to the first line: a + //message-redacted trace has no ':' on line one, and the first ':' would otherwise + //be a frame's line number. + private final static Pattern MSG_PATTERN = Pattern.compile("^([^\\r\\n:]*):[^\\r\\n]+"); public static String normalize(String stacktrace) { if (StringUtils.isBlank(stacktrace)) { @@ -46,7 +48,7 @@ public class EvalExceptionUtils { private static String trimMessage(String trace) { Matcher msgMatcher = MSG_PATTERN.matcher(trace); if (msgMatcher.find()) { - return msgMatcher.replaceFirst(""); + return msgMatcher.replaceFirst("$1"); } return trace; } diff --git a/tika-eval/tika-eval-core/src/test/java/org/apache/tika/eval/core/util/EvalExceptionUtilsTest.java b/tika-eval/tika-eval-core/src/test/java/org/apache/tika/eval/core/util/EvalExceptionUtilsTest.java new file mode 100644 index 0000000000..6e0ec6ca93 --- /dev/null +++ b/tika-eval/tika-eval-core/src/test/java/org/apache/tika/eval/core/util/EvalExceptionUtilsTest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.eval.core.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +public class EvalExceptionUtilsTest { + + @Test + public void fullTraceLosesMessagesOnly() { + String trace = "org.apache.tika.exception.TikaException: secret /path\n" + + "\tat org.apache.tika.Foo.bar(Foo.java:12)\n" + + "Caused by: java.io.IOException: other secret\n" + + "\tat org.apache.tika.Foo.baz(Foo.java:34)\n"; + assertEquals("o.a.t.exception.TikaException\n" + + "\tat o.a.t.Foo.bar(Foo.java:12)\n" + + "Caused by: java.io.IOException\n" + + "\tat o.a.t.Foo.baz(Foo.java:34)\n", EvalExceptionUtils.normalize(trace)); + } + + @Test + public void messageRedactedTraceNormalizesToSameKey() { + String full = "org.apache.tika.exception.TikaException: secret\n" + + "\tat org.apache.tika.Foo.bar(Foo.java:12)\n"; + String redacted = "org.apache.tika.exception.TikaException\n" + + "\tat org.apache.tika.Foo.bar(Foo.java:12)\n"; + assertEquals(EvalExceptionUtils.normalize(full), EvalExceptionUtils.normalize(redacted)); + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java index c99264fdc0..cd7eed583f 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java @@ -380,7 +380,7 @@ public class ZipParser extends AbstractArchiveParser { } catch (java.util.zip.ZipException ze) { // Truncated/corrupt central directory: stop iteration but keep // entries already extracted. Record the failure as a warning. - ParserUtils.recordParserFailure(this, ze, metadata); + ParserUtils.recordParserFailure(this, ze, metadata, context); break; } if (entry == null) { diff --git a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/WireRestrictedFetchEmitTupleTest.java b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/WireRestrictedFetchEmitTupleTest.java index 143bc9b5dd..d6868802ff 100644 --- a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/WireRestrictedFetchEmitTupleTest.java +++ b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/WireRestrictedFetchEmitTupleTest.java @@ -52,6 +52,15 @@ public class WireRestrictedFetchEmitTupleTest { (parseContextField.isEmpty() ? "" : "," + parseContextField) + "}"; } + @Test + public void pipesEndpointRejectsExceptionReporting() { + String ctx = "\"parse-context\":{\"exception-reporting\":{\"level\":\"FULL\"}}"; + Exception e = assertThrows(Exception.class, + () -> JsonFetchEmitTuple.fromJson(new StringReader(tuple(ctx)))); + assertTrue(root(e).contains("may not be supplied via a request parseContext"), + "expected wire-blocked rejection, got: " + root(e)); + } + @Test public void pipesEndpointRejectsParserInjection() { Exception e = assertThrows(Exception.class, diff --git a/tika-serialization/src/main/java/org/apache/tika/serialization/ComponentNameResolver.java b/tika-serialization/src/main/java/org/apache/tika/serialization/ComponentNameResolver.java index b78d8f14e3..91a1872d7b 100644 --- a/tika-serialization/src/main/java/org/apache/tika/serialization/ComponentNameResolver.java +++ b/tika-serialization/src/main/java/org/apache/tika/serialization/ComponentNameResolver.java @@ -24,6 +24,7 @@ import java.util.Set; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; +import org.apache.tika.config.ExceptionReporting; import org.apache.tika.config.loader.ComponentInfo; import org.apache.tika.config.loader.ComponentRegistry; import org.apache.tika.detect.Detector; @@ -90,6 +91,12 @@ public final class ComponentNameResolver { */ private static final Set<Class<?>> WIRE_BLOCKED_CONTEXT_KEYS = new HashSet<>(); + /** + * Plain config DTOs (not context-key interfaces) that a wire ParseContext may not set + * either: operator policy a caller must not be able to relax. + */ + private static final Set<Class<?>> WIRE_BLOCKED_CONFIG_KEYS = new HashSet<>(); + static { // Allowed: bounded to this request's metadata/output; no exec or IO. WIRE_INSTANTIABLE_CONTEXT_KEYS.add(MetadataFilter.class); @@ -106,6 +113,8 @@ public final class ComponentNameResolver { WIRE_BLOCKED_CONTEXT_KEYS.add(Renderer.class); WIRE_BLOCKED_CONTEXT_KEYS.add(Translator.class); WIRE_BLOCKED_CONTEXT_KEYS.add(EmbeddedDocumentExtractor.class); + + WIRE_BLOCKED_CONFIG_KEYS.add(ExceptionReporting.class); } private static final Map<String, ComponentRegistry> REGISTRIES = new ConcurrentHashMap<>(); @@ -310,11 +319,18 @@ public final class ComponentNameResolver { /** * True if a wire ParseContext must NOT instantiate this context-key type. Fail-closed: any * context-key interface not on the wire allowlist is blocked, so a newly-added one is refused - * until consciously allow-listed. Non-component keys (plain config DTOs) are never blocked. + * until consciously allow-listed. Plain config DTOs are blocked only if listed in + * {@link #WIRE_BLOCKED_CONFIG_KEYS}. */ public static boolean isWireBlocked(Class<?> contextKey) { - return CONTEXT_KEY_INTERFACES.contains(contextKey) - && !WIRE_INSTANTIABLE_CONTEXT_KEYS.contains(contextKey); + return WIRE_BLOCKED_CONFIG_KEYS.contains(contextKey) + || (CONTEXT_KEY_INTERFACES.contains(contextKey) + && !WIRE_INSTANTIABLE_CONTEXT_KEYS.contains(contextKey)); + } + + /** Config DTOs a wire ParseContext may not set; see {@link #WIRE_BLOCKED_CONFIG_KEYS}. */ + public static Set<Class<?>> getWireBlockedConfigKeys() { + return Collections.unmodifiableSet(WIRE_BLOCKED_CONFIG_KEYS); } /** Explicitly wire-blocked context-key interfaces; complement of the wire allowlist. */ diff --git a/tika-serialization/src/test/java/org/apache/tika/config/ExceptionReportingTest.java b/tika-serialization/src/test/java/org/apache/tika/config/ExceptionReportingTest.java new file mode 100644 index 0000000000..3b60a535ea --- /dev/null +++ b/tika-serialization/src/test/java/org/apache/tika/config/ExceptionReportingTest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.config.loader.TikaLoader; +import org.apache.tika.parser.ParseContext; + +public class ExceptionReportingTest extends TikaTest { + + @Test + public void testLoadFromConfig() throws Exception { + TikaLoader loader = TikaLoader.load(getConfigPath(getClass(), "exception-reporting-test.json")); + ParseContext context = loader.loadParseContext(); + ExceptionReporting reporting = ExceptionReporting.get(context); + assertEquals(new ExceptionReporting(ExceptionReporting.Level.MESSAGE_REDACTED, 500), reporting); + } + + @Test + public void testDefaults() { + assertSame(ExceptionReporting.DEFAULT, ExceptionReporting.get(null)); + assertSame(ExceptionReporting.DEFAULT, ExceptionReporting.get(new ParseContext())); + assertEquals(ExceptionReporting.Level.FULL, ExceptionReporting.DEFAULT.getLevel()); + assertEquals(ExceptionReporting.UNLIMITED, ExceptionReporting.DEFAULT.getMaxLength()); + } +} diff --git a/tika-serialization/src/test/java/org/apache/tika/serialization/WireRestrictedParseContextTest.java b/tika-serialization/src/test/java/org/apache/tika/serialization/WireRestrictedParseContextTest.java index 8f46e22dfb..bbfebac925 100644 --- a/tika-serialization/src/test/java/org/apache/tika/serialization/WireRestrictedParseContextTest.java +++ b/tika-serialization/src/test/java/org/apache/tika/serialization/WireRestrictedParseContextTest.java @@ -29,6 +29,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; import org.junit.jupiter.api.Test; +import org.apache.tika.config.ExceptionReporting; import org.apache.tika.config.loader.TikaObjectMapperFactory; import org.apache.tika.parser.ParseContext; import org.apache.tika.serialization.serdes.ParseContextDeserializer; @@ -86,6 +87,30 @@ public class WireRestrictedParseContextTest { "expected wire-blocked rejection, got: " + rootMessage(e)); } + @Test + public void restrictedRejectsExceptionReporting() { + // Operator policy on how much exception detail leaves the server; a caller must not be + // able to relax it per request. + String json = "{\"exception-reporting\":{\"level\":\"FULL\"}}"; + Exception e = assertThrows(Exception.class, + () -> restrictedMapper().readValue(json, ParseContext.class)); + assertTrue(rootMessage(e).contains("may not be supplied via a request parseContext"), + "expected wire-blocked rejection, got: " + rootMessage(e)); + } + + @Test + public void trustedAllowsExceptionReporting() throws Exception { + String json = "{\"exception-reporting\":{\"level\":\"REDACTED\",\"maxLength\":100}}"; + ObjectMapper mapper = TikaObjectMapperFactory.createMapper(); + SimpleModule module = new SimpleModule(); + module.addDeserializer(ParseContext.class, new ParseContextDeserializer(false)); + mapper.registerModule(module); + ParseContext ctx = mapper.readValue(json, ParseContext.class); + ParseContextUtils.resolveAll(ctx, ParseContextUtils.class.getClassLoader()); + assertEquals(new ExceptionReporting(ExceptionReporting.Level.REDACTED, 100), + ctx.get(ExceptionReporting.class)); + } + @Test public void restrictedAllowsFlatSelfConfiguringParserConfig() throws Exception { // Per-request tuning of an already-loaded self-configuring parser is config, not diff --git a/tika-serialization/src/test/resources/configs/exception-reporting-test.json b/tika-serialization/src/test/resources/configs/exception-reporting-test.json new file mode 100644 index 0000000000..f9515bc699 --- /dev/null +++ b/tika-serialization/src/test/resources/configs/exception-reporting-test.json @@ -0,0 +1,8 @@ +{ + "parse-context": { + "exception-reporting": { + "level": "MESSAGE_REDACTED", + "maxLength": 500 + } + } +}
