This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4856-presets in repository https://gitbox.apache.org/repos/asf/tika.git
commit a48580233aede6c28097f1cf17fa899d82d53150 Author: tallison <[email protected]> AuthorDate: Wed Sep 2 15:56:27 2026 -0400 TIKA-4856: resolve presets worker-side at config-tier trust --- CHANGES.txt | 9 +- .../ROOT/pages/using-tika/server/index.adoc | 19 ++- .../org/apache/tika/pipes/api/FetchEmitTuple.java | 20 +++- .../org/apache/tika/pipes/api/PipesResult.java | 1 + .../serialization/FetchEmitTupleDeserializer.java | 20 +++- .../serialization/FetchEmitTupleSerializer.java | 4 + .../pipes/core/serialization/PipesRequest.java | 2 +- .../tika/pipes/core/server/ConnectionHandler.java | 8 +- .../apache/tika/pipes/core/server/PipesServer.java | 41 ++++++- .../pipes/core/server/PresetNotFoundException.java | 31 +++++ .../pipes/core/server/SharedServerResources.java | 21 +++- .../core/serialization/JsonFetchEmitTupleTest.java | 27 +++++ .../tika/pipes/core/server/PresetMergeTest.java | 90 ++++++++++++++ .../apache/tika/config/loader/PresetRegistry.java | 131 +++++++++++++++++---- .../tika/config/loader/PresetRegistryTest.java | 92 +++++++++++++++ .../resources/test-presets/builtin-sample.json | 1 + .../server/core/resource/PipesParsingHelper.java | 28 ++++- .../tika/server/core/resource/PresetSelection.java | 28 +++++ .../core/resource/RecursiveMetadataResource.java | 6 + .../tika/server/core/resource/TikaResource.java | 57 +++++---- .../server/core/resource/UnpackerResource.java | 17 +++ .../org/apache/tika/server/core/CXFTestBase.java | 6 + .../tika/server/core/PresetEndpointsTest.java | 50 ++++++++ .../core/resource/TikaResourcePresetTest.java | 25 ++-- 24 files changed, 664 insertions(+), 70 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index c6a10a7791..d18d621da8 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -10,7 +10,14 @@ Release 4.1.0 - unreleased Exactly one preset per request, never combined with a config part, and usable without allowPerRequestConfig -- the preset routes are network- addressable separately from the /config endpoints, and nothing on the - classpath can activate a preset by itself. + classpath can activate a preset by itself. Only the preset's name + travels: the forked worker resolves the content from its own config at + config-tier trust, so presets may bind wire-blocked components and + raise timeout limits above the per-request clamp, and every active + preset resolves at startup or fails it. FetchEmitTuples submitted to + /pipes and /async may carry a "preset" field, resolved the same way + (unknown names answer the new PRESET_NOT_FOUND task status / HTTP 400; + the REST preset routes 404 before submission). * Raster previews for the vector thumbnails of Office documents: the new poi-metafile-renderer draws EMF and WMF images through POI (a PNG of diff --git a/docs/modules/ROOT/pages/using-tika/server/index.adoc b/docs/modules/ROOT/pages/using-tika/server/index.adoc index 6651effd64..80a4a3e234 100644 --- a/docs/modules/ROOT/pages/using-tika/server/index.adoc +++ b/docs/modules/ROOT/pages/using-tika/server/index.adoc @@ -142,7 +142,24 @@ preset is operator- or Tika-vetted, the preset routes do *not* require `allowPer they are the safe public knob, while free-form `/config` stays the privileged one. The `preset/{name}` path segment also gives network controls an addressable surface — a reverse proxy can allow `/rmeta/preset/render-thumbnails` (or all of `/rmeta/preset/`) while blocking -`/rmeta/config` entirely. An unknown preset name answers `404`. +`/rmeta/config` entirely. An unknown preset name answers `404`. Preset names may not start +with `config` (that path fragment gates the `/config` endpoints). + +Only the preset's *name* travels with a request: the forked parse worker resolves the content +from its own copy of the server config, with the same trust as the config's own +`parse-context` block. A preset can therefore configure components that per-request `/config` +input may not (detectors, embedded-document extraction, exception reporting, ...) and raise +timeout limits above the per-request clamp. Every active preset is fully resolved at startup, +so a preset that cannot resolve fails the server rather than its first request. + +Output format on the preset routes: an explicit format segment in the URL +(`/tika/preset/{name}/text`, `/rmeta/preset/{name}/xml`) always wins. Without one, a +`ContentHandlerFactory` the preset itself binds decides the format; otherwise the config's +factory, and finally the endpoint's usual default (Markdown). + +Presets also work on the batch surfaces: a `FetchEmitTuple` submitted to `/pipes` or `/async` +may carry a top-level `"preset": "name"` field, resolved the same way and overlaid beneath +whatever `parseContext` the tuple itself supplies. === `allowPipes` — the `/pipes` and `/async` endpoints diff --git a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/FetchEmitTuple.java b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/FetchEmitTuple.java index fbb2b94af9..16ac3023ae 100644 --- a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/FetchEmitTuple.java +++ b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/FetchEmitTuple.java @@ -38,6 +38,9 @@ public class FetchEmitTuple implements Serializable { private final Metadata metadata; private final ParseContext parseContext; private final ON_PARSE_EXCEPTION onParseException; + // Name of an operator-defined preset the server resolves from its own config + // at config-tier trust; only the name travels, never the preset's content. + private final String presetName; public FetchEmitTuple(String id, FetchKey fetchKey, EmitKey emitKey) { this(id, fetchKey, emitKey, new Metadata()); @@ -52,12 +55,18 @@ public class FetchEmitTuple implements Serializable { public FetchEmitTuple(String id, FetchKey fetchKey, EmitKey emitKey, Metadata metadata, ParseContext parseContext, ON_PARSE_EXCEPTION onParseException) { + this(id, fetchKey, emitKey, metadata, parseContext, onParseException, null); + } + + public FetchEmitTuple(String id, FetchKey fetchKey, EmitKey emitKey, Metadata metadata, ParseContext parseContext, + ON_PARSE_EXCEPTION onParseException, String presetName) { this.id = id; this.fetchKey = fetchKey; this.emitKey = emitKey; this.metadata = metadata; this.parseContext = parseContext; this.onParseException = onParseException; + this.presetName = presetName; } public String getId() { @@ -86,6 +95,11 @@ public class FetchEmitTuple implements Serializable { return onParseException; } + /** The selected preset's name, or null for none. */ + public String getPresetName() { + return presetName; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -98,7 +112,8 @@ public class FetchEmitTuple implements Serializable { FetchEmitTuple that = (FetchEmitTuple) o; return Objects.equals(id, that.id) && Objects.equals(fetchKey, that.fetchKey) && Objects.equals(emitKey, that.emitKey) && Objects.equals(metadata, that.metadata) && - Objects.equals(parseContext, that.parseContext) && onParseException == that.onParseException; + Objects.equals(parseContext, that.parseContext) && onParseException == that.onParseException && + Objects.equals(presetName, that.presetName); } @Override @@ -109,6 +124,7 @@ public class FetchEmitTuple implements Serializable { result = 31 * result + Objects.hashCode(metadata); result = 31 * result + Objects.hashCode(parseContext); result = 31 * result + Objects.hashCode(onParseException); + result = 31 * result + Objects.hashCode(presetName); return result; } @@ -116,6 +132,6 @@ public class FetchEmitTuple implements Serializable { public String toString() { return "FetchEmitTuple{" + "id='" + id + '\'' + ", fetchKey=" + fetchKey + ", emitKey=" + emitKey + ", metadata=" + metadata + ", parseContext=" + parseContext + - ", onParseException=" + onParseException + '}'; + ", onParseException=" + onParseException + ", presetName='" + presetName + "'}"; } } diff --git a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/PipesResult.java b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/PipesResult.java index fabbcd25ca..adf844b4a6 100644 --- a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/PipesResult.java +++ b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/PipesResult.java @@ -66,6 +66,7 @@ public record PipesResult(RESULT_STATUS status, EmitData emitData, String messag EMIT_EXCEPTION(CATEGORY.TASK_EXCEPTION), FETCHER_NOT_FOUND(CATEGORY.TASK_EXCEPTION), EMITTER_NOT_FOUND(CATEGORY.TASK_EXCEPTION), + PRESET_NOT_FOUND(CATEGORY.TASK_EXCEPTION), PAYLOAD_LIMIT_EXCEEDED(CATEGORY.TASK_EXCEPTION), // Process crashes - forked process died, auto-restart diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleDeserializer.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleDeserializer.java index 0b656dda94..3baf55647f 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleDeserializer.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleDeserializer.java @@ -26,6 +26,7 @@ import static org.apache.tika.pipes.core.serialization.FetchEmitTupleSerializer. import static org.apache.tika.pipes.core.serialization.FetchEmitTupleSerializer.ID; import static org.apache.tika.pipes.core.serialization.FetchEmitTupleSerializer.METADATA_KEY; import static org.apache.tika.pipes.core.serialization.FetchEmitTupleSerializer.ON_PARSE_EXCEPTION; +import static org.apache.tika.pipes.core.serialization.FetchEmitTupleSerializer.PRESET; import static org.apache.tika.serialization.serdes.ParseContextSerializer.PARSE_CONTEXT; import java.io.IOException; @@ -40,6 +41,7 @@ import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.JsonNode; +import org.apache.tika.config.loader.PresetRegistry; import org.apache.tika.metadata.Metadata; import org.apache.tika.parser.ParseContext; import org.apache.tika.pipes.api.ComponentIds; @@ -55,7 +57,7 @@ public class FetchEmitTupleDeserializer extends JsonDeserializer<FetchEmitTuple> private static final Set<String> KNOWN_KEYS = Set.of( ID, FETCHER, FETCH_KEY, EMITTER, EMIT_KEY, FETCH_RANGE_START, FETCH_RANGE_END, - METADATA_KEY, PARSE_CONTEXT, ON_PARSE_EXCEPTION); + METADATA_KEY, PARSE_CONTEXT, ON_PARSE_EXCEPTION, PRESET); private final boolean restricted; @@ -114,10 +116,11 @@ public class FetchEmitTupleDeserializer extends JsonDeserializer<FetchEmitTuple> ParseContext parseContext = parseContextNode == null ? new ParseContext() : ParseContextDeserializer.readParseContext(parseContextNode, true); FetchEmitTuple.ON_PARSE_EXCEPTION onParseException = readOnParseException(root); + String presetName = readPresetName(root); return new FetchEmitTuple(id, new FetchKey(fetcherId, fetchKey, fetchRangeStart, fetchRangeEnd), new EmitKey(emitterName, emitKey), metadata, parseContext, - onParseException); + onParseException, presetName); } /** @@ -147,6 +150,19 @@ public class FetchEmitTupleDeserializer extends JsonDeserializer<FetchEmitTuple> } } + /** + * A preset name is only a selector: the server resolves it against its own config, + * so the sole check here is the shared name syntax (which also bounds its length). + */ + private static String readPresetName(JsonNode root) throws IOException { + String presetName = readVal(PRESET, root, null, false); + if (presetName != null && !PresetRegistry.isValidName(presetName)) { + throw new IOException("invalid preset name (letters, digits, '.', '_', '-'; " + + "max 100 chars; may not start with 'config')"); + } + return presetName; + } + private static FetchEmitTuple.ON_PARSE_EXCEPTION readOnParseException(JsonNode root) throws IOException { JsonNode onParseExNode = root.get(ON_PARSE_EXCEPTION); if (onParseExNode == null) { diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleSerializer.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleSerializer.java index 608ab15f37..9667aa22c8 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleSerializer.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleSerializer.java @@ -40,6 +40,7 @@ public class FetchEmitTupleSerializer extends JsonSerializer<FetchEmitTuple> { public static final String EMIT_KEY = "emitKey"; public static final String METADATA_KEY = "metadata"; public static final String ON_PARSE_EXCEPTION = "onParseException"; + public static final String PRESET = "preset"; public void serialize(FetchEmitTuple t, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException { @@ -59,6 +60,9 @@ public class FetchEmitTupleSerializer extends JsonSerializer<FetchEmitTuple> { jsonGenerator.writeObjectField(METADATA_KEY, t.getMetadata()); } jsonGenerator.writeStringField(ON_PARSE_EXCEPTION, t.getOnParseException().name().toLowerCase(Locale.US)); + if (t.getPresetName() != null) { + jsonGenerator.writeStringField(PRESET, t.getPresetName()); + } ParseContext parseContext = t.getParseContext(); // Tailored: ParseContextSerializer's generic refusal suggests registering the // component -- for InlineBytes, exactly the forbidden fix. diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/PipesRequest.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/PipesRequest.java index 15745cb41a..4a6e41887b 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/PipesRequest.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/PipesRequest.java @@ -60,7 +60,7 @@ public final class PipesRequest { copy.copyFrom(ctx); copy.set(InlineBytes.class, null); FetchEmitTuple stripped = new FetchEmitTuple(t.getId(), t.getFetchKey(), t.getEmitKey(), - t.getMetadata(), copy, t.getOnParseException()); + t.getMetadata(), copy, t.getOnParseException(), t.getPresetName()); return new PipesRequest(stripped, inline.getBytes()); } diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ConnectionHandler.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ConnectionHandler.java index a1755be2ab..84ce8cf0fc 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ConnectionHandler.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ConnectionHandler.java @@ -165,7 +165,8 @@ public class ConnectionHandler implements Runnable, Closeable { } ParseContext mergedContext = null; try { - mergedContext = resources.createMergedParseContext(fetchEmitTuple.getParseContext()); + mergedContext = resources.createMergedParseContext( + fetchEmitTuple.getParseContext(), fetchEmitTuple.getPresetName()); ParseContextUtils.resolveAll(mergedContext, getClass().getClassLoader()); ServerProtocolIO.validateParseContext(mergedContext); ServerProtocolIO.clampRequestTimeoutLimits( @@ -184,6 +185,11 @@ public class ConnectionHandler implements Runnable, Closeable { executorCompletionService.submit(pipesWorker); loopUntilDone(fetchEmitTuple, mergedContext, intermediateResult, countDownLatch, parseTimeout); + } catch (PresetNotFoundException e) { + // caller error, not a server fault: answer it and keep serving + LOG.warn("handlerId={}: id={}: {}", handlerId, fetchEmitTuple.getId(), e.getMessage()); + protocolIO.writeFinished(new PipesResult( + PipesResult.RESULT_STATUS.PRESET_NOT_FOUND, e.getMessage())); } catch (TikaConfigException e) { LOG.error("handlerId={}: config error processing request", handlerId, e); handleCrash(PipesMessageType.UNSPECIFIED_CRASH, fetchEmitTuple.getId(), e); diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java index 173a7efc6e..e496ca7f8f 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java @@ -48,6 +48,7 @@ import org.xml.sax.SAXException; import org.apache.tika.config.ExceptionReporting; import org.apache.tika.config.ParseTimeout; import org.apache.tika.config.TimeoutLimits; +import org.apache.tika.config.loader.PresetRegistry; import org.apache.tika.config.loader.TikaJsonConfig; import org.apache.tika.config.loader.TikaLoader; import org.apache.tika.detect.Detector; @@ -185,6 +186,7 @@ public class PipesServer implements AutoCloseable { private RecursiveParserWrapper rMetaParser; private FetcherManager fetcherManager; private EmitterManager emitterManager; + private PresetRegistry presetRegistry; private ConfigStore configStore; private final ExecutorService executorService = Executors.newSingleThreadExecutor(); private final ExecutorCompletionService<PipesResult> executorCompletionService = new ExecutorCompletionService<>(executorService); @@ -441,7 +443,8 @@ public class PipesServer implements AutoCloseable { ParseContext mergedContext; ParseTimeout parseTimeout; try { - mergedContext = createMergedParseContext(fetchEmitTuple.getParseContext()); + mergedContext = createMergedParseContext( + fetchEmitTuple.getParseContext(), fetchEmitTuple.getPresetName()); ParseContextUtils.resolveAll(mergedContext, getClass().getClassLoader()); ServerProtocolIO.validateParseContext(mergedContext); ServerProtocolIO.clampRequestTimeoutLimits( @@ -454,6 +457,12 @@ public class PipesServer implements AutoCloseable { // ParseTimeout.getOrCreate(mergedContext) call (inside CompositeParser) // sees this instance rather than racing to install its own. parseTimeout = ParseTimeout.getOrCreate(mergedContext); + } catch (PresetNotFoundException e) { + // caller error, not a server fault: answer it and keep serving + LOG.warn("id={}: {}", fetchEmitTuple.getId(), e.getMessage()); + writeFinished(new PipesResult( + PipesResult.RESULT_STATUS.PRESET_NOT_FOUND, e.getMessage())); + break; } catch (Exception e) { // write the reason to the client instead of a bare exit code handleCrash(PipesMessageType.UNSPECIFIED_CRASH, fetchEmitTuple.getId(), e); @@ -719,7 +728,8 @@ public class PipesServer implements AutoCloseable { this.autoDetectParser = (AutoDetectParser) tikaLoader.loadAutoDetectParser(); this.detector = this.autoDetectParser.getDetector(); this.rMetaParser = new RecursiveParserWrapper(autoDetectParser); - + // fails startup on an unresolvable preset, mirroring the front-end's own load + this.presetRegistry = PresetRegistry.load(tikaJsonConfig, tikaLoader.getClassLoader()); } /** @@ -729,9 +739,11 @@ public class PipesServer implements AutoCloseable { * Creates a fresh context each time to avoid shared state between requests. * * @param requestContext the ParseContext from FetchEmitTuple - * @return a new ParseContext with defaults + request overrides + * @param presetName name of the preset to overlay at config-tier trust, or null + * @return a new ParseContext with defaults + preset + request overrides */ - private ParseContext createMergedParseContext(ParseContext requestContext) throws TikaConfigException { + private ParseContext createMergedParseContext(ParseContext requestContext, String presetName) + throws TikaConfigException { // Create fresh context with defaults from tika-config (e.g., DigesterFactory) ParseContext mergedContext = tikaLoader.loadParseContext(); // EmbeddedDocumentExtractor is deliberately left unset here: setting a default (even @@ -740,12 +752,31 @@ public class PipesServer implements AutoCloseable { // no-ops whenever one is already bound), silently disabling embedded content // extraction for every non-UNPACK parse mode. UNPACK mode sets its own // EmbeddedDocumentExtractor + UnpackedByteCount in PipesWorker's UNPACK-mode setup. - // Request-level values override config defaults + mergePreset(presetRegistry, presetName, mergedContext); + // Request-level values override config defaults and the preset mergedContext.copyFrom(requestContext); seedCacheMemoryBudget(mergedContext); return mergedContext; } + /** + * Overlays the named preset, resolved from this server's own config at config-tier + * trust. Only the name arrived on the wire; the caller's untrusted delta is copied + * on top afterwards and remains subject to the wire screens and timeout clamping. + */ + static void mergePreset(PresetRegistry registry, String presetName, ParseContext merged) + throws TikaConfigException { + if (presetName == null) { + return; + } + ParseContext presetContext = registry.newParseContext(presetName); + if (presetContext == null) { + throw new PresetNotFoundException( + "No preset named '" + presetName + "' is active in this server's config"); + } + merged.copyFrom(presetContext); + } + private ConfigStore createConfigStore(PipesConfig pipesConfig, TikaPluginManager tikaPluginManager) throws TikaException { String configStoreType = pipesConfig.getConfigStoreType(); String configStoreParams = pipesConfig.getConfigStoreParams(); diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PresetNotFoundException.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PresetNotFoundException.java new file mode 100644 index 0000000000..b3198b223f --- /dev/null +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PresetNotFoundException.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.pipes.core.server; + +import org.apache.tika.exception.TikaConfigException; + +/** + * A request selected a preset name this server's config does not activate. A caller + * error, not a server fault: answered with a {@code PRESET_NOT_FOUND} result rather + * than the crash path other pre-parse failures take. + */ +public class PresetNotFoundException extends TikaConfigException { + + public PresetNotFoundException(String msg) { + super(msg); + } +} diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/SharedServerResources.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/SharedServerResources.java index 21371d2ca9..5c97818673 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/SharedServerResources.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/SharedServerResources.java @@ -21,6 +21,7 @@ import java.io.IOException; import org.xml.sax.SAXException; import org.apache.tika.config.ExceptionReporting; +import org.apache.tika.config.loader.PresetRegistry; import org.apache.tika.config.loader.TikaJsonConfig; import org.apache.tika.config.loader.TikaLoader; import org.apache.tika.detect.Detector; @@ -66,6 +67,7 @@ public class SharedServerResources { private final EmitStrategy emitStrategy; private final ConfigStore configStore; private final ExceptionReporting exceptionReporting; + private final PresetRegistry presetRegistry; private SharedServerResources(TikaLoader tikaLoader, PipesConfig pipesConfig, AutoDetectParser autoDetectParser, Detector detector, @@ -74,7 +76,8 @@ public class SharedServerResources { ContentHandlerFactory defaultContentHandlerFactory, MetadataWriteLimiterFactory defaultMetadataWriteLimiterFactory, EmitStrategy emitStrategy, ConfigStore configStore, - ExceptionReporting exceptionReporting) { + ExceptionReporting exceptionReporting, + PresetRegistry presetRegistry) { this.tikaLoader = tikaLoader; this.pipesConfig = pipesConfig; this.autoDetectParser = autoDetectParser; @@ -88,6 +91,7 @@ public class SharedServerResources { this.emitStrategy = emitStrategy; this.configStore = configStore; this.exceptionReporting = exceptionReporting; + this.presetRegistry = presetRegistry; } /** @@ -127,10 +131,14 @@ public class SharedServerResources { EmitStrategy emitStrategy = pipesConfig.getEmitStrategy().getType(); + // fails startup on an unresolvable preset, mirroring the front-end's own load + PresetRegistry presetRegistry = + PresetRegistry.load(tikaJsonConfig, tikaLoader.getClassLoader()); + return new SharedServerResources(tikaLoader, pipesConfig, autoDetectParser, detector, rMetaParser, fetcherManager, emitterManager, metadataFilter, contentHandlerFactory, metadataWriteLimiterFactory, emitStrategy, configStore, - ExceptionReporting.get(configContext)); + ExceptionReporting.get(configContext), presetRegistry); } private static ConfigStore createConfigStore(PipesConfig pipesConfig, TikaPluginManager tikaPluginManager) @@ -155,14 +163,17 @@ public class SharedServerResources { * Creates a merged ParseContext with defaults from tika-config overlaid with request values. * * @param requestContext the ParseContext from FetchEmitTuple - * @return a new ParseContext with defaults + request overrides + * @param presetName name of the preset to overlay at config-tier trust, or null + * @return a new ParseContext with defaults + preset + request overrides */ - public ParseContext createMergedParseContext(ParseContext requestContext) throws TikaConfigException { + public ParseContext createMergedParseContext(ParseContext requestContext, String presetName) + throws TikaConfigException { ParseContext mergedContext = tikaLoader.loadParseContext(); // EmbeddedDocumentExtractor is deliberately left unset here -- see PipesServer's // createMergedParseContext for why defaulting it would silently disable embedded // content extraction for every non-UNPACK parse mode. - // Request-level values override config defaults + PipesServer.mergePreset(presetRegistry, presetName, mergedContext); + // Request-level values override config defaults and the preset mergedContext.copyFrom(requestContext); PipesServer.seedCacheMemoryBudget(mergedContext); return mergedContext; diff --git a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/JsonFetchEmitTupleTest.java b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/JsonFetchEmitTupleTest.java index f029c3e4a0..abba24cb22 100644 --- a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/JsonFetchEmitTupleTest.java +++ b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/serialization/JsonFetchEmitTupleTest.java @@ -18,6 +18,8 @@ package org.apache.tika.pipes.core.serialization; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.io.Reader; import java.io.StringReader; @@ -157,4 +159,29 @@ public class JsonFetchEmitTupleTest { assertEquals(unpackConfig.getSuffixStrategy(), deserializedConfig.getSuffixStrategy(), "suffixStrategy should be preserved"); } + + @Test + public void testPresetNameRoundTrips() throws Exception { + FetchEmitTuple t = new FetchEmitTuple("my_id", new FetchKey("my_fetcher", "k"), + new EmitKey("my_emitter", "e"), new Metadata(), new ParseContext(), + FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT, "ocr-heavy"); + FetchEmitTuple deserialized = + JsonFetchEmitTuple.fromJson(new StringReader(JsonFetchEmitTuple.toJson(t))); + assertEquals("ocr-heavy", deserialized.getPresetName()); + + FetchEmitTuple noPreset = new FetchEmitTuple("my_id", new FetchKey("my_fetcher", "k"), + new EmitKey("my_emitter", "e")); + assertNull(JsonFetchEmitTuple + .fromJson(new StringReader(JsonFetchEmitTuple.toJson(noPreset))) + .getPresetName()); + } + + @Test + public void testInvalidPresetNameRefused() { + // the name is only a selector, but it must obey the shared syntax bound + assertThrows(Exception.class, () -> JsonFetchEmitTuple.fromJson(new StringReader( + "{\"id\":\"i\",\"fetcher\":\"f\",\"fetchKey\":\"k\",\"preset\":\"../etc\"}"))); + assertThrows(Exception.class, () -> JsonFetchEmitTuple.fromJson(new StringReader( + "{\"id\":\"i\",\"fetcher\":\"f\",\"fetchKey\":\"k\",\"preset\":\"config-x\"}"))); + } } diff --git a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/server/PresetMergeTest.java b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/server/PresetMergeTest.java new file mode 100644 index 0000000000..6be2dbf8fe --- /dev/null +++ b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/server/PresetMergeTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.pipes.core.server; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import org.apache.tika.config.TimeoutLimits; +import org.apache.tika.config.loader.PresetRegistry; +import org.apache.tika.config.loader.TikaJsonConfig; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.serialization.ParseContextUtils; + +/** + * Preset content is operator config resolved at config-tier trust in the worker: + * a preset's timeout limits must survive {@code clampRequestTimeoutLimits} (only + * request-supplied limits are clamped), and an unknown name is a task-level error, + * not a crash. + */ +public class PresetMergeTest { + + @TempDir + Path tmp; + + private PresetRegistry registry(String configJson) throws Exception { + Path p = tmp.resolve("config-" + configJson.hashCode() + ".json"); + Files.writeString(p, configJson); + return PresetRegistry.load(TikaJsonConfig.load(p), getClass().getClassLoader()); + } + + @Test + public void testPresetTimeoutLimitsAreNotClamped() throws Exception { + PresetRegistry registry = registry(""" + {"presets": {"slow-ocr": {"timeout-limits": {"totalTaskTimeoutMillis": 3600000}}}} + """); + ParseContext merged = new ParseContext(); + PipesServer.mergePreset(registry, "slow-ocr", merged); + ParseContext requestContext = new ParseContext(); + merged.copyFrom(requestContext); + ParseContextUtils.resolveAll(merged, getClass().getClassLoader()); + + // the clamp fires only on request-supplied limits; the preset's ride at config tier + ServerProtocolIO.clampRequestTimeoutLimits(requestContext, merged, 60_000); + assertEquals(3600000, TimeoutLimits.get(merged).getTotalTaskTimeoutMillis()); + } + + @Test + public void testRequestLimitsStillClampedOverPreset() throws Exception { + PresetRegistry registry = registry(""" + {"presets": {"slow-ocr": {"timeout-limits": {"totalTaskTimeoutMillis": 3600000}}}} + """); + ParseContext merged = new ParseContext(); + PipesServer.mergePreset(registry, "slow-ocr", merged); + ParseContext requestContext = new ParseContext(); + requestContext.setJsonConfig("timeout-limits", + "{\"totalTaskTimeoutMillis\": 7200000}"); + merged.copyFrom(requestContext); + ParseContextUtils.resolveAll(merged, getClass().getClassLoader()); + + ServerProtocolIO.clampRequestTimeoutLimits(requestContext, merged, 60_000); + assertEquals(60_000, TimeoutLimits.get(merged).getTotalTaskTimeoutMillis()); + } + + @Test + public void testUnknownPresetIsTaskLevelError() throws Exception { + PresetRegistry registry = registry("{}"); + assertThrows(PresetNotFoundException.class, + () -> PipesServer.mergePreset(registry, "nope", new ParseContext())); + } +} diff --git a/tika-serialization/src/main/java/org/apache/tika/config/loader/PresetRegistry.java b/tika-serialization/src/main/java/org/apache/tika/config/loader/PresetRegistry.java index 88301659c5..c4892392f6 100644 --- a/tika-serialization/src/main/java/org/apache/tika/config/loader/PresetRegistry.java +++ b/tika-serialization/src/main/java/org/apache/tika/config/loader/PresetRegistry.java @@ -22,6 +22,7 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.Enumeration; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; @@ -32,6 +33,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.tika.exception.TikaConfigException; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.ContentHandlerFactory; +import org.apache.tika.serialization.ParseContextUtils; +import org.apache.tika.serialization.serdes.ParseContextDeserializer; /** * Named, vetted parse-context fragments a caller can select whole ("presets"). @@ -40,6 +45,12 @@ import org.apache.tika.exception.TikaConfigException; * by name only, so the configuration itself stays in Tika and in the server's * config rather than in consuming applications. * <p> + * A preset is operator-authored config, not caller input: it is resolved with + * the same trust as the config's own {@code parse-context} block (no wire-block + * screening), and only its <em>name</em> ever travels on a request. Every + * active preset is fully resolved at load time, so a preset that cannot + * resolve fails startup rather than its first request. + * <p> * Nothing is active unless the config's {@code presets} block names it: an * entry with value {@code true} activates the catalog definition of that name * (content shipped on the classpath, so it tracks the Tika version); an object @@ -67,10 +78,15 @@ public final class PresetRegistry { // Names ride in URL paths and config keys private static final Pattern NAME = Pattern.compile("[A-Za-z0-9][A-Za-z0-9._-]{0,99}"); - private final Map<String, String> presets; + private final Map<String, JsonNode> presets; + private final Set<String> withContentHandlerFactory; + private final ClassLoader classLoader; - private PresetRegistry(Map<String, String> presets) { + private PresetRegistry(Map<String, JsonNode> presets, Set<String> withContentHandlerFactory, + ClassLoader classLoader) { this.presets = presets; + this.withContentHandlerFactory = withContentHandlerFactory; + this.classLoader = classLoader; } /** @@ -78,14 +94,18 @@ public final class PresetRegistry { * names it lists are active. {@code true} activates a catalog definition * (startup error if the catalog has no such name); an object defines the * preset in place; {@code false}/{@code null} deactivates explicitly. + * Every active preset is resolved here, so a preset whose content cannot + * bind is a startup error. * * @param config the loaded config, may be null (empty roster) - * @param classLoader loader to scan for catalog preset indexes, may be null - * for the thread context loader + * @param classLoader loader to scan for catalog preset indexes and resolve + * preset components, may be null for the thread context loader */ public static PresetRegistry load(TikaJsonConfig config, ClassLoader classLoader) throws TikaConfigException { - Map<String, String> presets = new LinkedHashMap<>(); + ClassLoader loader = classLoader != null ? classLoader + : Thread.currentThread().getContextClassLoader(); + Map<String, JsonNode> presets = new LinkedHashMap<>(); if (config != null && config.hasKey(CONFIG_KEY)) { JsonNode block = config.getRootNode().get(CONFIG_KEY); if (block == null || !block.isObject()) { @@ -93,18 +113,16 @@ public final class PresetRegistry { "'" + CONFIG_KEY + "' must be an object of preset definitions"); } // load the inert catalog only when the config can reference it - ClassLoader loader = classLoader != null ? classLoader - : Thread.currentThread().getContextClassLoader(); - Map<String, String> catalog = loadCatalog(loader); + Map<String, JsonNode> catalog = loadCatalog(loader); Iterator<Map.Entry<String, JsonNode>> fields = block.fields(); while (fields.hasNext()) { Map.Entry<String, JsonNode> e = fields.next(); String name = e.getKey(); JsonNode value = e.getValue(); if (value.isNull() || (value.isBoolean() && !value.asBoolean())) { - presets.remove(name); + continue; // explicit no-op } else if (value.isBoolean()) { - String content = catalog.get(name); + JsonNode content = catalog.get(name); if (content == null) { throw new TikaConfigException("preset '" + name + "': true activates a catalog preset, but no catalog " + @@ -112,26 +130,54 @@ public final class PresetRegistry { } presets.put(validName(name), content); } else if (value.isObject()) { - presets.put(validName(name), value.toString()); + presets.put(validName(name), value); } else { throw new TikaConfigException("preset '" + name + "' must be an " + "object, true (activate catalog definition), or false/null"); } } } - return new PresetRegistry(presets); + Set<String> withContentHandlerFactory = new HashSet<>(); + for (Map.Entry<String, JsonNode> e : presets.entrySet()) { + ParseContext resolved = resolve(e.getKey(), e.getValue(), loader); + if (resolved.get(ContentHandlerFactory.class) != null) { + withContentHandlerFactory.add(e.getKey()); + } + } + return new PresetRegistry(presets, withContentHandlerFactory, loader); } - private static Map<String, String> loadCatalog(ClassLoader loader) + /** + * Trusted-tier resolution: presets are operator config, so no wire-block screening + * -- identical treatment to the config's own {@code parse-context} block. + */ + private static ParseContext resolve(String name, JsonNode content, ClassLoader loader) throws TikaConfigException { - Map<String, String> presets = new LinkedHashMap<>(); - ObjectMapper mapper = new ObjectMapper(); + try { + ParseContext context = ParseContextDeserializer.readParseContext(content, false); + ParseContextUtils.resolveAll(context, loader); + return context; + } catch (IOException | TikaConfigException e) { + throw new TikaConfigException( + "preset '" + name + "' failed to resolve: " + e.getMessage(), e); + } + } + + private static Map<String, JsonNode> loadCatalog(ClassLoader loader) + throws TikaConfigException { + Map<String, JsonNode> presets = new LinkedHashMap<>(); + Map<String, URL> sources = new LinkedHashMap<>(); + // Same JSON dialect as the config itself (comments allowed, duplicate keys refused) + ObjectMapper mapper = TikaObjectMapperFactory.getMapper(); try { Enumeration<URL> indexes = loader.getResources(INDEX_RESOURCE); while (indexes.hasMoreElements()) { URL index = indexes.nextElement(); - for (String line : new String(index.openStream().readAllBytes(), - StandardCharsets.UTF_8).split("\n")) { + String indexContent; + try (InputStream is = index.openStream()) { + indexContent = new String(is.readAllBytes(), StandardCharsets.UTF_8); + } + for (String line : indexContent.split("\n")) { line = line.trim(); if (line.isEmpty() || line.startsWith("#")) { continue; @@ -154,7 +200,14 @@ public final class PresetRegistry { throw new TikaConfigException("preset '" + name + "' must contain a JSON object: " + resource); } - presets.put(name, content.toString()); + JsonNode previous = presets.put(name, content); + // classpath order is not a config statement: refuse a silent last-wins + if (previous != null && !previous.equals(content)) { + throw new TikaConfigException("catalog preset '" + name + + "' is defined with different content by " + + sources.get(name) + " and " + index); + } + sources.put(name, index); } } } @@ -169,22 +222,58 @@ public final class PresetRegistry { } private static String validName(String name) throws TikaConfigException { - if (!NAME.matcher(name).matches()) { + if (!isValidName(name)) { throw new TikaConfigException("invalid preset name (letters, digits, " + - "'.', '_', '-'; max 100 chars): '" + name + "'"); + "'.', '_', '-'; max 100 chars; may not start with 'config', which is " + + "reserved so preset URL routes stay distinct from /config endpoint " + + "gating): '" + name + "'"); } return name; } + /** + * True if {@code name} is a legal preset name: the character/length rule above, and + * not starting with "config" (tika-server gates {@code /config} endpoints on that + * path fragment, so such a name would be unreachable there). Public so wire + * deserializers can bound a preset-name field with the same rule. + */ + public static boolean isValidName(String name) { + return name != null && NAME.matcher(name).matches() + && !name.regionMatches(true, 0, "config", 0, 6); + } + public Set<String> names() { return Collections.unmodifiableSet(presets.keySet()); } + public boolean hasPreset(String name) { + return name != null && presets.containsKey(name); + } + /** * The preset's content -- a {@code parse-context}-shaped JSON object of * component configurations -- or null if no preset has this name. */ public String parseContextJson(String name) { - return name == null ? null : presets.get(name); + JsonNode node = name == null ? null : presets.get(name); + return node == null ? null : node.toString(); + } + + /** + * A fresh, fully resolved ParseContext for the named preset, or null if no preset + * has this name. Fresh per call: callers mutate the result per request. + */ + public ParseContext newParseContext(String name) throws TikaConfigException { + JsonNode content = name == null ? null : presets.get(name); + return content == null ? null : resolve(name, content, classLoader); + } + + /** + * True if the named preset binds a {@link ContentHandlerFactory}: a route with no + * explicit format segment should then leave the choice to the preset rather than + * forcing its own default. + */ + public boolean suppliesContentHandlerFactory(String name) { + return name != null && withContentHandlerFactory.contains(name); } } diff --git a/tika-serialization/src/test/java/org/apache/tika/config/loader/PresetRegistryTest.java b/tika-serialization/src/test/java/org/apache/tika/config/loader/PresetRegistryTest.java index 14d93eb5fc..308fc97c58 100644 --- a/tika-serialization/src/test/java/org/apache/tika/config/loader/PresetRegistryTest.java +++ b/tika-serialization/src/test/java/org/apache/tika/config/loader/PresetRegistryTest.java @@ -22,6 +22,8 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.net.URL; +import java.net.URLClassLoader; import java.nio.file.Files; import java.nio.file.Path; @@ -30,7 +32,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.apache.tika.config.ExceptionReporting; import org.apache.tika.exception.TikaConfigException; +import org.apache.tika.parser.ParseContext; public class PresetRegistryTest { @@ -119,6 +123,94 @@ public class PresetRegistryTest { """)); } + @Test + public void testConfigPrefixedNameRejected() { + // tika-server gates /config endpoints on the path fragment, so such a + // preset would be unreachable there; refuse it at definition time + assertThrows(TikaConfigException.class, () -> load(""" + {"presets": {"config-fast": {}}} + """)); + assertThrows(TikaConfigException.class, () -> load(""" + {"presets": {"CONFIGX": {}}} + """)); + assertTrue(PresetRegistry.isValidName("fast-config")); + } + + @Test + public void testUnresolvablePresetFailsLoad() { + // a known component with malformed content must fail at load, not first use + assertThrows(TikaConfigException.class, () -> load(""" + {"presets": {"bad": {"basic-content-handler-factory": {"type": "NO_SUCH_TYPE"}}}} + """)); + } + + @Test + public void testNewParseContextIsTrustedAndFresh() throws Exception { + // exception-reporting is wire-blocked for caller-supplied contexts; a preset is + // operator config and must be able to bind it + PresetRegistry registry = load(""" + {"presets": {"reporting": {"exception-reporting": {"maxLength": 512}}}} + """); + ParseContext first = registry.newParseContext("reporting"); + assertTrue(first.get(ExceptionReporting.class) != null); + // fresh per call: callers mutate the result per request + assertTrue(first != registry.newParseContext("reporting")); + assertNull(registry.newParseContext("nope")); + assertNull(registry.newParseContext(null)); + } + + @Test + public void testSuppliesContentHandlerFactory() throws Exception { + PresetRegistry registry = load(""" + {"presets": { + "with-chf": {"basic-content-handler-factory": {"type": "XML"}}, + "without-chf": {"embedded-limits": {"maxDepth": 2}}}} + """); + assertTrue(registry.suppliesContentHandlerFactory("with-chf")); + assertFalse(registry.suppliesContentHandlerFactory("without-chf")); + assertFalse(registry.suppliesContentHandlerFactory("nope")); + assertFalse(registry.suppliesContentHandlerFactory(null)); + } + + @Test + public void testCatalogNameCollisionAcrossJarsFails() throws Exception { + Path dirA = catalogDir("a", "colliding", "{\"embedded-limits\": {\"maxDepth\": 1}}"); + Path dirB = catalogDir("b", "colliding", "{\"embedded-limits\": {\"maxDepth\": 2}}"); + // parent is the test loader, so component classes still resolve; its own + // catalog contributes only the distinct builtin-sample name + try (URLClassLoader loader = new URLClassLoader( + new URL[]{dirA.toUri().toURL(), dirB.toUri().toURL()}, + getClass().getClassLoader())) { + assertThrows(TikaConfigException.class, () -> PresetRegistry.load( + config("{\"presets\": {\"colliding\": true}}"), loader)); + } + } + + @Test + public void testCatalogIdenticalDuplicateTolerated() throws Exception { + // the same jar visible twice on a classpath is noise, not a conflict + Path dirA = catalogDir("a2", "dup", "{\"embedded-limits\": {\"maxDepth\": 3}}"); + Path dirB = catalogDir("b2", "dup", "{\"embedded-limits\": {\"maxDepth\": 3}}"); + try (URLClassLoader loader = new URLClassLoader( + new URL[]{dirA.toUri().toURL(), dirB.toUri().toURL()}, + getClass().getClassLoader())) { + PresetRegistry registry = PresetRegistry.load( + config("{\"presets\": {\"dup\": true}}"), loader); + assertTrue(registry.hasPreset("dup")); + } + } + + private Path catalogDir(String dirName, String presetName, String json) throws Exception { + Path dir = tmp.resolve(dirName); + Files.createDirectories(dir.resolve("META-INF/tika")); + // resource path unique per dir: identical paths would shadow on the classpath + Files.writeString(dir.resolve("META-INF/tika/presets.idx"), + presetName + "=/presets-" + dirName + "/" + presetName + ".json\n"); + Files.createDirectories(dir.resolve("presets-" + dirName)); + Files.writeString(dir.resolve("presets-" + dirName + "/" + presetName + ".json"), json); + return dir; + } + @Test public void testNonObjectPresetRejected() { assertThrows(TikaConfigException.class, () -> load(""" diff --git a/tika-serialization/src/test/resources/test-presets/builtin-sample.json b/tika-serialization/src/test/resources/test-presets/builtin-sample.json index c28c46063d..938e7a78d5 100644 --- a/tika-serialization/src/test/resources/test-presets/builtin-sample.json +++ b/tika-serialization/src/test/resources/test-presets/builtin-sample.json @@ -1,3 +1,4 @@ { + // catalog files use the same JSON dialect as the config: comments allowed "basic-content-handler-factory": {"type": "TEXT"} } diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java index 9c1c1765b8..5732661194 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java @@ -202,6 +202,8 @@ public class PipesParsingHelper { // Set parse mode in context parseContext.set(ParseMode.class, parseMode); + String presetName = liftPresetSelection(parseContext); + // This parser is shared with /pipes, whose own default is EMIT_ALL. No // emitter is configured for /tika/rmeta/unpack requests (EmitKey.NO_EMIT // below) -- results must come back over the socket, so set PASSBACK_ALL @@ -213,7 +215,9 @@ public class PipesParsingHelper { fetchKey, EmitKey.NO_EMIT, metadata, - parseContext + parseContext, + FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT, + presetName ); // Execute parse via pipes - results will be passed back through socket @@ -241,6 +245,20 @@ public class PipesParsingHelper { } } + /** + * Lifts the in-process {@link PresetSelection} carrier out of the context and onto the + * tuple's own preset field: only the name travels; the forked worker resolves the + * preset's content from its own config at config-tier trust. + */ + private static String liftPresetSelection(ParseContext parseContext) { + PresetSelection preset = parseContext.get(PresetSelection.class); + if (preset == null) { + return null; + } + parseContext.set(PresetSelection.class, null); + return preset.name(); + } + /** Longest suffix carried over from a client filename; keeps well clear of NAME_MAX. */ private static final int MAX_SUFFIX_LENGTH = 20; @@ -418,7 +436,7 @@ public class PipesParsingHelper { // The caller named a fetcher/emitter this server does not have. Nothing failed // on our side, and retrying the same request will never succeed -- 500 told // clients to retry a request that is permanently malformed. - case FETCHER_NOT_FOUND, EMITTER_NOT_FOUND -> + case FETCHER_NOT_FOUND, EMITTER_NOT_FOUND, PRESET_NOT_FOUND -> Response.Status.BAD_REQUEST; case PAYLOAD_LIMIT_EXCEEDED -> Response.Status.REQUEST_ENTITY_TOO_LARGE; @@ -566,6 +584,8 @@ public class PipesParsingHelper { // Set parse mode to UNPACK parseContext.set(ParseMode.class, ParseMode.UNPACK); + String presetName = liftPresetSelection(parseContext); + // Shared parser (see parse() above) -- PASSBACK_ALL is also required here // for correctness: with UNPACK mode, EmitHandler.shouldEmit() only skips // re-emitting metadata (already emitted as part of the zip) when the @@ -602,7 +622,9 @@ public class PipesParsingHelper { fetchKey, emitKey, metadata, - parseContext + parseContext, + FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT, + presetName ); // Execute parse via pipes diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PresetSelection.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PresetSelection.java new file mode 100644 index 0000000000..264c1bc262 --- /dev/null +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PresetSelection.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.server.core.resource; + +/** + * In-process carrier for a request's selected preset name, riding the request + * ParseContext between the resource and {@link PipesParsingHelper}, which lifts it + * onto the tuple's own preset field before serialization. Never travels on the wire + * itself (the wire serializer refuses unregistered context entries, so a leak fails + * loudly). The preset's content is resolved by the forked worker from its own config + * at config-tier trust. + */ +public record PresetSelection(String name) { +} diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java index a8fe598bab..b3126b4f7d 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/RecursiveMetadataResource.java @@ -161,6 +161,12 @@ public class RecursiveMetadataResource { @PathParam(HANDLER_TYPE_PARAM) String handlerTypeName) throws Exception { ParseContext context = tikaResource.createPresetContext(presetName); + // An explicit format segment wins over a factory the preset itself binds; with no + // segment, parseMetadataWithContext defers to the preset's factory, then the + // config's, then the endpoint default. + if (handlerTypeName != null && !handlerTypeName.isBlank()) { + tikaResource.setupContentHandlerFactory(context, handlerTypeName); + } Metadata metadata = tikaResource.newRequestMetadata(); try (TikaInputStream tis = TikaInputStream.get(is)) { fillMetadata(null, metadata, httpHeaders.getRequestHeaders()); diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java index 67ab84ad62..1033d1324c 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java @@ -124,32 +124,28 @@ public class TikaResource { this.presetRegistry = PresetRegistry.load(tikaLoader.getConfig(), tikaLoader.getClassLoader()); } catch (TikaConfigException e) { - // config error: fail startup, not the first preset request + // config error (including a preset that cannot resolve): fail startup, + // not the first preset request throw new IllegalStateException("Invalid 'presets' configuration", e); } } /** - * A request context with the named preset's parse-context fragment merged in. + * A request context carrying the named preset selection. Only the name is recorded + * here: the forked worker resolves the preset from its own copy of this config at + * config-tier trust, so preset content is never treated as caller-supplied wire + * data (which would screen out wire-blocked components and clamp its timeouts). * A preset is selected whole and exclusively -- the {@code preset} routes take * no config part, so it never combines with request-supplied configuration. * * @throws NotFoundException if no preset has this name */ public ParseContext createPresetContext(String presetName) { - String fragment = presetRegistry.parseContextJson(presetName); - if (fragment == null) { + if (!presetRegistry.hasPreset(presetName)) { throw new NotFoundException("No such preset: " + presetName); } ParseContext context = createRequestContext(); - try { - mergeParseContextFromConfig(fragment, context); - } catch (IOException | TikaConfigException e) { - // the preset came from Tika or the server config, so this is a - // server-side configuration error, not a caller error - throw new WebApplicationException( - "Preset '" + presetName + "' failed to resolve: " + e.getMessage(), 500); - } + context.set(PresetSelection.class, new PresetSelection(presetName)); return context; } @@ -501,11 +497,19 @@ public class TikaResource { * @param handlerTypeName the handler type name */ public void setupContentHandlerFactoryIfNeeded(ParseContext context, String handlerTypeName) { + if (context.get(ContentHandlerFactory.class) != null) { + return; + } + // A selected preset that binds its own factory decides the format on routes with no + // explicit format segment; the worker resolves it from the preset at config tier. + PresetSelection preset = context.get(PresetSelection.class); + if (preset != null && presetRegistry.suppliesContentHandlerFactory(preset.name())) { + return; + } // A config-declared factory still takes precedence; it is no longer visible in the // request context, so leaving the context untouched lets the worker resolve it from // the same config. - if (context.get(ContentHandlerFactory.class) == null - && !configSuppliesContentHandlerFactory) { + if (!configSuppliesContentHandlerFactory) { setupContentHandlerFactory(context, handlerTypeName); } } @@ -624,27 +628,40 @@ public class TikaResource { // address /tika/preset/* -- or a single preset -- independently of /tika/config*. // These routes take no config part; a preset never combines with request config. + // explicitHandlerType semantics: non-null (an explicit format segment in the URL) wins + // over everything, including a factory the preset itself binds; null defers to the + // preset's factory, then the config's, then the endpoint default. + private Response putRawPreset(InputStream is, HttpHeaders httpHeaders, String presetName, - String handlerTypeName) throws IOException { + String explicitHandlerType) throws IOException { ParseContext context = createPresetContext(presetName); Metadata metadata = newRequestMetadata(); fillMetadata(null, metadata, httpHeaders.getRequestHeaders()); + if (explicitHandlerType != null) { + setupContentHandlerFactory(context, explicitHandlerType); + } try (TikaInputStream tis = TikaInputStream.get(is)) { - return produceRawOutputWithContext(tis, metadata, context, handlerTypeName); + return produceRawOutputWithContext(tis, metadata, context, explicitHandlerType); } } private Metadata putJsonPreset(InputStream is, HttpHeaders httpHeaders, String presetName, - String handlerTypeName) throws IOException { + String explicitHandlerType) throws IOException { ParseContext context = createPresetContext(presetName); Metadata metadata = newRequestMetadata(); fillMetadata(null, metadata, httpHeaders.getRequestHeaders()); + if (explicitHandlerType != null) { + setupContentHandlerFactory(context, explicitHandlerType); + } try (TikaInputStream tis = TikaInputStream.get(is)) { - return produceJsonWithContext(tis, metadata, context, handlerTypeName); + return produceJsonWithContext(tis, metadata, context, explicitHandlerType); } } - /** As the bare /tika endpoint (Markdown), with the named preset applied. */ + /** + * As the bare /tika endpoint, with the named preset applied. A factory the preset + * binds decides the output format here; without one the Markdown default applies. + */ @PUT @Consumes("*/*") @Produces("text/plain;charset=UTF-8") @@ -652,7 +669,7 @@ public class TikaResource { public Response getDefaultWithPreset(final InputStream is, @Context HttpHeaders httpHeaders, @PathParam("presetName") String presetName) throws IOException { - return putRawPreset(is, httpHeaders, presetName, "md"); + return putRawPreset(is, httpHeaders, presetName, null); } /** As /tika/text, with the named preset applied. */ diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java index 148a86e239..4acaa959d5 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java @@ -145,10 +145,24 @@ public class UnpackerResource { * @param info URI info * @return streaming zip response */ + /** + * The wildcard {@code id} on the pre-existing routes would otherwise silently absorb a + * transposed preset URL ({@code /unpack/all/preset/x}, {@code /unpack/preset} with no + * name) and run with no preset applied -- a silent wrong-config success. + */ + private static void rejectPresetInWildcard(UriInfo info) { + String id = info.getPathParameters().getFirst("id"); + if (id != null && (id.equals("/preset") || id.startsWith("/preset/"))) { + throw new jakarta.ws.rs.NotFoundException( + "preset routes are PUT /unpack/preset/{name}[/all]"); + } + } + @jakarta.ws.rs.Path("/{id:(/.*)?}") @PUT @Produces("application/zip") public Response unpack(InputStream is, @Context HttpHeaders httpHeaders, @Context UriInfo info) throws Exception { + rejectPresetInWildcard(info); ParseContext pc = tikaResource.createRequestContext(); Metadata metadata = tikaResource.newRequestMetadata(); try (TikaInputStream tis = TikaInputStream.get(is)) { @@ -172,6 +186,7 @@ public class UnpackerResource { @Consumes("multipart/form-data") @Produces("application/zip") public Response unpackWithConfig(List<Attachment> attachments, @Context HttpHeaders httpHeaders, @Context UriInfo info) throws Exception { + rejectPresetInWildcard(info); ParseContext pc = tikaResource.createRequestContext(); Metadata metadata = tikaResource.newRequestMetadata(); try (TikaInputStream tis = tikaResource.setupMultipartConfig(attachments, metadata, pc)) { @@ -193,6 +208,7 @@ public class UnpackerResource { @PUT @Produces("application/zip") public Response unpackAll(InputStream is, @Context HttpHeaders httpHeaders, @Context UriInfo info) throws Exception { + rejectPresetInWildcard(info); ParseContext pc = tikaResource.createRequestContext(); Metadata metadata = tikaResource.newRequestMetadata(); try (TikaInputStream tis = TikaInputStream.get(is)) { @@ -216,6 +232,7 @@ public class UnpackerResource { @Consumes("multipart/form-data") @Produces("application/zip") public Response unpackAllWithConfig(List<Attachment> attachments, @Context HttpHeaders httpHeaders, @Context UriInfo info) throws Exception { + rejectPresetInWildcard(info); ParseContext pc = tikaResource.createRequestContext(); Metadata metadata = tikaResource.newRequestMetadata(); try (TikaInputStream tis = tikaResource.setupMultipartConfig(attachments, metadata, pc)) { diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/CXFTestBase.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/CXFTestBase.java index b7a158363f..155b7dcc87 100644 --- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/CXFTestBase.java +++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/CXFTestBase.java @@ -357,6 +357,12 @@ public abstract class CXFTestBase { if (metadataFilters != null && !metadataFilters.isEmpty()) { root.set("metadata-filters", metadataFilters); } + // The worker resolves preset names from its own config, so presets must be + // visible there just like the parse-context defaults above. + JsonNode presets = tikaConfig.get("presets"); + if (presets != null && !presets.isEmpty()) { + root.set("presets", presets); + } } catch (Exception e) { LOG.debug("Could not carry config into the worker config: {}", e.getMessage()); } diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/PresetEndpointsTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/PresetEndpointsTest.java index 0325dc910c..dc703751e0 100644 --- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/PresetEndpointsTest.java +++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/PresetEndpointsTest.java @@ -18,6 +18,7 @@ package org.apache.tika.server.core; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import java.io.ByteArrayInputStream; import java.io.InputStream; @@ -59,6 +60,10 @@ public class PresetEndpointsTest extends CXFTestBase { ObjectNode presets = config.putObject("presets"); presets.putObject("xml-content") .putObject("basic-content-handler-factory").put("type", "XML"); + // exception-reporting is wire-blocked for caller-supplied config; a preset is + // operator config and must be able to bind it (resolved worker-side) + presets.putObject("reporting") + .putObject("exception-reporting").put("maxLength", 512); return new ByteArrayInputStream( MAPPER.writeValueAsString(config).getBytes(UTF_8)); } @@ -131,6 +136,51 @@ public class PresetEndpointsTest extends CXFTestBase { assertContains("<body><p>hello world</p>", content); } + @Test + public void testExplicitFormatSegmentWinsOverPresetFactory() throws Exception { + // /tika/preset/xml-content/text: the URL's own format segment beats the XML + // factory the preset binds -- it rides the request delta, which the worker + // overlays on top of the preset + Response response = WebClient + .create(endPoint + "/tika/preset/xml-content/text") + .accept("text/plain") + .put(ClassLoader.getSystemResourceAsStream(HELLO_WORLD)); + assertEquals(200, response.getStatus()); + String content = getStringFromInputStream((InputStream) response.getEntity()); + assertContains("hello world", content); + assertFalse(content.contains("<body>"), "explicit /text segment must win: " + content); + } + + @Test + public void testWireBlockedComponentWorksInPreset() throws Exception { + // previously this 500'd: the preset was pushed through the untrusted wire + // deserializer, which refuses exception-reporting + Response response = WebClient + .create(endPoint + "/tika/preset/reporting") + .accept("text/plain") + .put(ClassLoader.getSystemResourceAsStream(HELLO_WORLD)); + assertEquals(200, response.getStatus()); + assertContains("hello world", + getStringFromInputStream((InputStream) response.getEntity())); + } + + @Test + public void testTransposedUnpackPresetUrlIs404() throws Exception { + // /unpack/all/preset/{name} would otherwise fall into the /all{id} wildcard + // and run with no preset applied -- a silent wrong-config success + Response response = WebClient + .create(endPoint + "/unpack/all/preset/xml-content") + .accept("application/zip") + .put(ClassLoader.getSystemResourceAsStream(HELLO_WORLD)); + assertEquals(404, response.getStatus()); + + response = WebClient + .create(endPoint + "/unpack/preset") + .accept("application/zip") + .put(ClassLoader.getSystemResourceAsStream(HELLO_WORLD)); + assertEquals(404, response.getStatus()); + } + @Test public void testTikaUnknownPresetIs404() throws Exception { Response response = WebClient diff --git a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/resource/TikaResourcePresetTest.java b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/resource/TikaResourcePresetTest.java index 2f296defe9..61363e86d7 100644 --- a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/resource/TikaResourcePresetTest.java +++ b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/resource/TikaResourcePresetTest.java @@ -17,6 +17,7 @@ package org.apache.tika.server.core.resource; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import java.nio.file.Files; @@ -28,7 +29,6 @@ import org.junit.jupiter.api.io.TempDir; import org.apache.tika.config.loader.TikaLoader; import org.apache.tika.parser.ParseContext; -import org.apache.tika.sax.BasicContentHandlerFactory; import org.apache.tika.sax.ContentHandlerFactory; import org.apache.tika.server.core.ServerStatus; @@ -54,12 +54,14 @@ public class TikaResourcePresetTest { } @Test - public void testPresetExpandsIntoRequestContext() throws Exception { + public void testPresetSelectionRidesRequestContextByNameOnly() throws Exception { + // only the name is recorded: the forked worker resolves the content from its + // own config at config-tier trust, so nothing preset-shaped may enter the + // request (caller-tier) context here ParseContext context = newTikaResource(CONFIG, true).createPresetContext("xml-content"); - BasicContentHandlerFactory chf = - (BasicContentHandlerFactory) context.get(ContentHandlerFactory.class); - assertEquals(BasicContentHandlerFactory.HANDLER_TYPE.XML, chf.getType()); + assertEquals("xml-content", context.get(PresetSelection.class).name()); + assertNull(context.get(ContentHandlerFactory.class)); } @Test @@ -68,9 +70,7 @@ public class TikaResourcePresetTest { // free-form per-request-config privilege ParseContext context = newTikaResource(CONFIG, false).createPresetContext("xml-content"); - BasicContentHandlerFactory chf = - (BasicContentHandlerFactory) context.get(ContentHandlerFactory.class); - assertEquals(BasicContentHandlerFactory.HANDLER_TYPE.XML, chf.getType()); + assertEquals("xml-content", context.get(PresetSelection.class).name()); } @Test @@ -84,4 +84,13 @@ public class TikaResourcePresetTest { assertThrows(IllegalStateException.class, () -> newTikaResource("{\"presets\": {\"bad\": \"a string\"}}", true)); } + + @Test + public void testUnresolvablePresetFailsStartup() { + // preset content is resolved at load: a malformed component fails startup, + // not the first preset request + assertThrows(IllegalStateException.class, () -> newTikaResource(""" + {"presets": {"bad": {"basic-content-handler-factory": {"type": "NOPE"}}}} + """, true)); + } }
