This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4808-pre-4.0.0-release-fixes in repository https://gitbox.apache.org/repos/asf/tika.git
commit d8625f0a4b4ca7ed1b1695b846b12e3f3597a967 Author: tallison <[email protected]> AuthorDate: Mon Aug 17 06:27:48 2026 -0400 TIKA-4808: config isolation, plugin boundary by scope, stream resource hardening --- CHANGES.txt | 8 ++- docs/modules/ROOT/pages/advanced/spooling.adoc | 4 +- .../pages/migration-to-4x/migrating-to-4x.adoc | 11 +++- .../ROOT/pages/pipes/plugins/writing-a-plugin.adoc | 21 ++---- tika-core/src/main/java/org/apache/tika/Tika.java | 42 +++++++----- .../org/apache/tika/config/ParseContextConfig.java | 27 ++++---- .../java/org/apache/tika/config/package-info.java | 2 +- .../java/org/apache/tika/detect/package-info.java | 2 +- .../java/org/apache/tika/digest/DigestHelper.java | 6 +- .../org/apache/tika/exception/package-info.java | 2 +- .../org/apache/tika/extractor/package-info.java | 2 +- .../java/org/apache/tika/io/ByteArraySource.java | 6 +- .../org/apache/tika/io/CachingInputStream.java | 5 ++ .../java/org/apache/tika/io/CachingSource.java | 29 +++++--- .../main/java/org/apache/tika/io/FileSource.java | 2 +- .../main/java/org/apache/tika/io/StreamCache.java | 40 +++++++---- .../java/org/apache/tika/io/TikaInputSource.java | 2 +- .../java/org/apache/tika/io/TikaInputStream.java | 66 +++++++++---------- .../main/java/org/apache/tika/io/package-info.java | 2 +- .../java/org/apache/tika/metadata/ISO19115.java | 2 +- .../java/org/apache/tika/metadata/Metadata.java | 4 +- .../org/apache/tika/metadata/package-info.java | 2 +- .../java/org/apache/tika/mime/package-info.java | 2 +- .../main/java/org/apache/tika/package-info.java | 2 +- .../java/org/apache/tika/parser/ParsingIntent.java | 25 +++---- .../java/org/apache/tika/parser/package-info.java | 2 +- .../tika/sax/BasicContentHandlerFactory.java | 3 +- .../java/org/apache/tika/sax/package-info.java | 2 +- .../org/apache/tika/sax/xpath/package-info.java | 2 +- .../java/org/apache/tika/utils/package-info.java | 2 +- .../tika/metadata/metadata-migration-3x-4x.json | 2 +- .../apache/tika/TikaDetectStreamOwnershipTest.java | 77 ++++++++++++++++++++++ .../org/apache/tika/io/TikaInputStreamTest.java | 61 +++++++++++++++-- .../apache/tika/metadata/metadata-key-fields.json | 2 +- .../org/apache/tika/metadata/metadata-keys.json | 2 +- .../org/apache/tika/metadata/migration-overlay.tsv | 2 +- tika-parent/pom.xml | 2 +- .../geoinfo/GeographicInformationParser.java | 2 +- tika-parsers/tika-parsers-ml/tika-vlm/pom.xml | 7 ++ .../tika/parser/RecursiveParserWrapperTest.java | 7 +- .../java/org/apache/tika/parser/pdf/PDFParser.java | 11 ++-- .../apache/tika/parser/pkg/CompressorParser.java | 14 ++-- .../detect/zip/DefaultZipContainerDetector.java | 2 +- .../org/apache/tika/async/cli/TikaAsyncCLI.java | 4 +- .../tika/pipes/core/extractor/UnpackExtractor.java | 5 +- tika-pipes/tika-pipes-plugins/pom.xml | 41 ++++++++++++ .../tika-pipes-atlassian-jwt/pom.xml | 6 +- .../fetcher/atlassianjwt/AtlassianJwtFetcher.java | 16 ++++- .../tika-pipes-plugins/tika-pipes-az-blob/pom.xml | 9 --- .../tika-pipes-plugins/tika-pipes-csv/pom.xml | 9 --- .../tika-pipes-plugins/tika-pipes-es/pom.xml | 4 -- .../tika-pipes-file-system/pom.xml | 9 --- .../tika-pipes-plugins/tika-pipes-gcs/pom.xml | 9 --- .../tika-pipes-google-drive/pom.xml | 6 +- .../fetcher/googledrive/GoogleDriveFetcher.java | 3 +- .../tika-pipes-plugins/tika-pipes-http/pom.xml | 4 -- .../tika/pipes/fetcher/http/HttpFetcher.java | 12 +++- .../fetcher/http/config/HttpFetcherConfig.java | 3 +- .../tika-pipes-plugins/tika-pipes-jdbc/pom.xml | 9 --- .../pipes/reporter/jdbc/JDBCPipesReporter.java | 2 + .../tika-pipes-plugins/tika-pipes-json/pom.xml | 11 +--- .../tika-pipes-plugins/tika-pipes-kafka/pom.xml | 9 --- .../tika-pipes-microsoft-graph/pom.xml | 4 -- .../tika-pipes-opensearch/pom.xml | 9 --- .../tika-pipes-plugins/tika-pipes-s3/pom.xml | 9 --- .../apache/tika/pipes/fetcher/s3/S3Fetcher.java | 2 + .../tika-pipes-plugins/tika-pipes-solr/pom.xml | 9 --- .../config/loader/AbstractSpiComponentLoader.java | 14 +++- .../tika/serialization/ConfigDeserializer.java | 7 +- .../apache/tika/config/loader/TikaLoaderTest.java | 15 +++++ .../tika/serialization/ConfigDeserializerTest.java | 38 +++++++++++ .../test-default-parser-unknown-exclusion.json | 9 +++ .../tika/server/core/resource/TikaResource.java | 12 ++-- .../server/core/writer/CSVMessageBodyWriter.java | 2 +- 74 files changed, 512 insertions(+), 307 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index ddf54b492c..46975e6b85 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -126,7 +126,11 @@ Release 4.0.0 - ??? TikaInputStream. Every third-party Parser, Detector or EmbeddedDocumentExtractor implementation must be updated; callers can wrap with TikaInputStream.get(...). The Tika facade (Tika.parse/parseToString) - still accepts an InputStream and is unaffected + still accepts an InputStream and is unaffected. Tika.detect(InputStream, ...) + no longer resets the stream to its original position: detection now reads + ahead through a TikaInputStream, so on return the caller's stream must be + treated as consumed. It still does not close the caller's stream, and any + temporary file spooled during detection is deleted before it returns (TIKA-4399, TIKA-4541, TIKA-4569). * TikaConfig and the org.apache.tika.config XML-configuration API are @@ -195,7 +199,7 @@ Release 4.0.0 - ??? AmazonTranscribeConfig, MagikaDetector/SiegfriedDetector configs) (TIKA-4758). - * TikaInputStream no longer buffers by default. A stream is consumed in + * TikaInputStream no longer caches by default. A stream is consumed in passthrough mode unless enableRewind() is called at position 0; rewind()/getFile()/getPath() after reading without enableRewind() throw instead of silently spooling. Digesters call enableRewind() themselves diff --git a/docs/modules/ROOT/pages/advanced/spooling.adoc b/docs/modules/ROOT/pages/advanced/spooling.adoc index 630706eda4..04b60f95b8 100644 --- a/docs/modules/ROOT/pages/advanced/spooling.adoc +++ b/docs/modules/ROOT/pages/advanced/spooling.adoc @@ -64,7 +64,9 @@ File file = tis.getFile(); * **Initialized with `Path`**: The file is used directly for random access. No spooling needed. * **Initialized with `byte[]`**: The bytes are kept in memory. Spooling only on demand. * **Initialized with `InputStream`**: `getPath()` or `getFile()` spools the stream to a - temporary file, which is automatically cleaned up when the stream is closed. + temporary file, which is automatically cleaned up when the stream is closed. Spooling + requires position 0 (or a prior `enableRewind()` at position 0); reading first and + spooling later throws. ==== Benefits of Decentralized Spooling diff --git a/docs/modules/ROOT/pages/migration-to-4x/migrating-to-4x.adoc b/docs/modules/ROOT/pages/migration-to-4x/migrating-to-4x.adoc index 7b8583e1c6..86661acde1 100644 --- a/docs/modules/ROOT/pages/migration-to-4x/migrating-to-4x.adoc +++ b/docs/modules/ROOT/pages/migration-to-4x/migrating-to-4x.adoc @@ -56,12 +56,21 @@ try (TikaInputStream tis = TikaInputStream.get(myInputStream)) { ---- The `Tika` facade (`Tika#parse`, `Tika#parseToString`, `Tika#detect`) still accepts a plain -`InputStream` and is unaffected — if you only use the facade, nothing changes. +`InputStream`. One behavior change: `Tika#detect(InputStream, ...)` no longer resets the stream +to its original position — treat the stream as consumed after the call. It still does not close +the caller's stream. Why: passing `TikaInputStream` explicitly makes the spooling and rewind contract visible in the signature instead of leaving every implementation to wrap defensively. See xref:advanced/spooling.adoc[spooling] for the stream contract. +`TikaInputStream` itself dropped several members in 4.0: `isTikaInputStream`, `cast`, +`getPath(int)`, `get(InputStreamFactory)` (both overloads), `hasInputStreamFactory`, +`getInputStreamFactory`, and `setOpenContainer(Object, long)` — along with the +`InputStreamFactory` class. `get(byte[])` and `getFromContainer` no longer declare +`IOException`. `enableRewind()` now throws a checked `IOException` when bytes have +already been read. + [#thin-launcher-zip] == `tika-app` and `tika-server` distributions: jar -> zip diff --git a/docs/modules/ROOT/pages/pipes/plugins/writing-a-plugin.adoc b/docs/modules/ROOT/pages/pipes/plugins/writing-a-plugin.adoc index f17674d81f..3bc6f3299a 100644 --- a/docs/modules/ROOT/pages/pipes/plugins/writing-a-plugin.adoc +++ b/docs/modules/ROOT/pages/pipes/plugins/writing-a-plugin.adoc @@ -219,16 +219,10 @@ If that file is missing or empty, the plugin loads and contributes nothing. === Collecting runtime dependencies -`maven-dependency-plugin` copies runtime dependencies to `target/lib`, minus the boundary -artifacts: - -[source,xml] ----- -<properties> - <plugin.excluded.artifactIds>tika-core,tika-pipes-api,tika-serialization,tika-plugins-core</plugin.excluded.artifactIds> - <plugin.excluded.groupIds>org.apache.logging.log4j,org.slf4j</plugin.excluded.groupIds> -</properties> ----- +`maven-dependency-plugin` copies runtime dependencies to `target/lib`. There is no +exclusion list to maintain: every boundary artifact is declared `provided`, and +`includeScope=runtime` excludes `provided` by definition. A boundary artifact can +only end up in `lib/` if a pom re-declares it at compile scope -- do not. [source,xml] ---- @@ -243,8 +237,6 @@ artifacts: <configuration> <outputDirectory>${project.build.directory}/lib</outputDirectory> <includeScope>runtime</includeScope> - <excludeArtifactIds>${plugin.excluded.artifactIds}</excludeArtifactIds> - <excludeGroupIds>${plugin.excluded.groupIds}</excludeGroupIds> </configuration> </execution> </executions> @@ -276,7 +268,7 @@ A plugin's classloader prefers the plugin's own classes over the host's. So if y `MyFetcherFactory` implements a `FetcherFactory` that is a *different* `Class` object from the host's. The host then finds no extensions, or fails casting one. -That is what the `plugin.excluded.artifactIds` property above prevents, and it is the usual cause +That is what the `provided` scoping above prevents, and it is the usual cause of a plugin that loads cleanly and then behaves as if it were not there — or of a `NoClassDefFoundError` / `ClassCastException` naming a Tika type. @@ -372,7 +364,8 @@ java -Dtika.plugin.dev.mode=true ... * `plugin.properties` at the zip root, with `plugin.id` and `plugin.class`. * `@Extension` on the factory, never on the fetcher or emitter. * `META-INF/extensions.idx` non-empty in `target/classes` after compiling. -* `tika-core`, `tika-pipes-api`, `tika-plugins-core`, `tika-serialization`, `pf4j` and the logging +* `tika-core`, `tika-pipes-api`, `tika-plugins-core`, `tika-serialization`, + `tika-pipes-core`, `tika-pipes-iterator-commons`, `pf4j` and the logging implementations `provided`, and absent from `lib/`. * The zip — not an unpacked directory — dropped in a `plugin-roots` directory. * `getName()` unique against every other loaded plugin. diff --git a/tika-core/src/main/java/org/apache/tika/Tika.java b/tika-core/src/main/java/org/apache/tika/Tika.java index 5195cf9108..5e5a4d913c 100644 --- a/tika-core/src/main/java/org/apache/tika/Tika.java +++ b/tika-core/src/main/java/org/apache/tika/Tika.java @@ -30,6 +30,7 @@ import org.apache.tika.detect.DefaultDetector; import org.apache.tika.detect.Detector; import org.apache.tika.exception.TikaException; import org.apache.tika.exception.WriteLimitReachedException; +import org.apache.tika.io.TemporaryResources; import org.apache.tika.io.TikaInputStream; import org.apache.tika.language.translate.DefaultTranslator; import org.apache.tika.language.translate.Translator; @@ -129,12 +130,14 @@ public class Tika { * in which case only the given document metadata is used for type * detection. * <p> - * If the document stream supports the - * {@link InputStream#markSupported() mark feature}, then the stream is - * marked and reset to the original position before this method returns. - * Only a limited number of bytes are read from the stream. + * The given document stream is <em>not</em> closed by this method, and any + * temporary file spooled during detection is deleted before it returns. * <p> - * The given document stream is <em>not</em> closed by this method. + * The stream <em>is</em> consumed: detection reads ahead through a + * {@link TikaInputStream}, so on return the caller's stream must be treated as + * advanced, not rewound. Callers that need to read the document afterwards should + * pass a {@link TikaInputStream} they own (which they can + * {@link TikaInputStream#rewind() rewind}) or re-open the source. * <p> * Unlike in the {@link #parse(InputStream, Metadata)} method, the * given document metadata is <em>not</em> modified by this method. @@ -148,7 +151,10 @@ public class Tika { if (stream == null) { return detector.detect(null, metadata, new ParseContext()).toString(); } - try (TikaInputStream tis = TikaInputStream.get(stream)) { + // Dispose only what detection spooled, never the caller's stream. Closing the + // TikaInputStream would also dispose a caller-supplied one (get() returns it as-is). + try (TemporaryResources tmp = new TemporaryResources()) { + TikaInputStream tis = TikaInputStream.get(stream, tmp, metadata); return detector.detect(tis, metadata, new ParseContext()).toString(); } } @@ -158,12 +164,14 @@ public class Tika { * based on the content of the given document stream and the name of the * document. * <p> - * If the document stream supports the - * {@link InputStream#markSupported() mark feature}, then the stream is - * marked and reset to the original position before this method returns. - * Only a limited number of bytes are read from the stream. + * The given document stream is <em>not</em> closed by this method, and any + * temporary file spooled during detection is deleted before it returns. * <p> - * The given document stream is <em>not</em> closed by this method. + * The stream <em>is</em> consumed: detection reads ahead through a + * {@link TikaInputStream}, so on return the caller's stream must be treated as + * advanced, not rewound. Callers that need to read the document afterwards should + * pass a {@link TikaInputStream} they own (which they can + * {@link TikaInputStream#rewind() rewind}) or re-open the source. * * @param stream the document stream * @param name document name @@ -182,12 +190,14 @@ public class Tika { * Detects the media type of the given document. The type detection is * based on the content of the given document stream. * <p> - * If the document stream supports the - * {@link InputStream#markSupported() mark feature}, then the stream is - * marked and reset to the original position before this method returns. - * Only a limited number of bytes are read from the stream. + * The given document stream is <em>not</em> closed by this method, and any + * temporary file spooled during detection is deleted before it returns. * <p> - * The given document stream is <em>not</em> closed by this method. + * The stream <em>is</em> consumed: detection reads ahead through a + * {@link TikaInputStream}, so on return the caller's stream must be treated as + * advanced, not rewound. Callers that need to read the document afterwards should + * pass a {@link TikaInputStream} they own (which they can + * {@link TikaInputStream#rewind() rewind}) or re-open the source. * * @param stream the document stream * @return detected media type diff --git a/tika-core/src/main/java/org/apache/tika/config/ParseContextConfig.java b/tika-core/src/main/java/org/apache/tika/config/ParseContextConfig.java index 8f1dc03371..a08e501585 100644 --- a/tika-core/src/main/java/org/apache/tika/config/ParseContextConfig.java +++ b/tika-core/src/main/java/org/apache/tika/config/ParseContextConfig.java @@ -69,14 +69,10 @@ public class ParseContextConfig { /** * Retrieves runtime configuration from ParseContext. * <p> - * This method first checks if the config is already resolved in ParseContext - * (via {@code context.get(configClass)}). If found, it returns immediately without - * re-deserializing. This is efficient for embedded documents where the config - * was already deserialized for the parent document. - * <p> - * If not found, it checks jsonConfigs for the config key and deserializes - * the JSON. The deserialized config is cached in resolvedConfigs and also - * set in the main ParseContext for future lookups. + * Resolution is per config key: the resolved-config cache, then JSON config + * (deserialized and cached). A class-keyed entry set programmatically via + * {@code context.set(configClass, config)} is honored only when no JSON config + * exists for the key, since several components may share one config class. * <p> * This method performs defensive checking: if the ParseContext has JSON configuration * for the requested key but the ConfigDeserializer is not available on the classpath, @@ -98,16 +94,17 @@ public class ParseContextConfig { return defaultConfig; } - // First check if config is already resolved in ParseContext - // (may have been set by a previous call or by user code) - T existingConfig = context.get(configClass); - if (existingConfig != null) { - return existingConfig; + // Per-key resolution first: a class-keyed entry may belong to a different + // component that happens to share this config class. + T resolved = context.getResolvedConfig(configKey); + if (resolved != null) { + return resolved; } - // Check for JSON config if (!context.hasJsonConfig(configKey)) { - return defaultConfig; + // no JSON for this key: honor a programmatic, class-keyed override + T existingConfig = context.get(configClass); + return existingConfig != null ? existingConfig : defaultConfig; } // JSON config exists for this key - ConfigDeserializer MUST be available diff --git a/tika-core/src/main/java/org/apache/tika/config/package-info.java b/tika-core/src/main/java/org/apache/tika/config/package-info.java index 77a0559f7f..61ee292ef7 100644 --- a/tika-core/src/main/java/org/apache/tika/config/package-info.java +++ b/tika-core/src/main/java/org/apache/tika/config/package-info.java @@ -18,5 +18,5 @@ /** * Tika configuration tools. */ [email protected]("1.0.0") [email protected]("2.0.0") package org.apache.tika.config; diff --git a/tika-core/src/main/java/org/apache/tika/detect/package-info.java b/tika-core/src/main/java/org/apache/tika/detect/package-info.java index dede49cfb6..a579b63597 100644 --- a/tika-core/src/main/java/org/apache/tika/detect/package-info.java +++ b/tika-core/src/main/java/org/apache/tika/detect/package-info.java @@ -18,5 +18,5 @@ /** * Media type detection. */ [email protected]("1.0.0") [email protected]("2.0.0") package org.apache.tika.detect; diff --git a/tika-core/src/main/java/org/apache/tika/digest/DigestHelper.java b/tika-core/src/main/java/org/apache/tika/digest/DigestHelper.java index 07bc9c4e6f..a5f751b1ae 100644 --- a/tika-core/src/main/java/org/apache/tika/digest/DigestHelper.java +++ b/tika-core/src/main/java/org/apache/tika/digest/DigestHelper.java @@ -80,10 +80,8 @@ public class DigestHelper { Digester digester = digesterFactory.build(); - // Handle embedded stream translation if needed (e.g., for OLE2 objects in TikaInputStream's - // open container). The translator consumes `tis` to produce the translated bytes that get - // digested, so we enableRewind() before and rewind() after — otherwise the caller (typically - // AutoDetectParser) would see an exhausted stream and fail its zero-byte probe (TIKA-XXXX). + // The translator consumes `tis` (e.g. OLE2), so enableRewind() before and rewind() + // after -- otherwise the caller would see an exhausted stream. if (EMBEDDED_STREAM_TRANSLATOR.shouldTranslate(tis, metadata)) { tis.enableRewind(); try (TemporaryResources tmp = new TemporaryResources()) { diff --git a/tika-core/src/main/java/org/apache/tika/exception/package-info.java b/tika-core/src/main/java/org/apache/tika/exception/package-info.java index 80ab125814..13c7b98c8d 100644 --- a/tika-core/src/main/java/org/apache/tika/exception/package-info.java +++ b/tika-core/src/main/java/org/apache/tika/exception/package-info.java @@ -18,5 +18,5 @@ /** * Tika exception. */ [email protected]("1.0.0") [email protected]("2.0.0") package org.apache.tika.exception; diff --git a/tika-core/src/main/java/org/apache/tika/extractor/package-info.java b/tika-core/src/main/java/org/apache/tika/extractor/package-info.java index 3d3e92b525..ddb1dfd33c 100644 --- a/tika-core/src/main/java/org/apache/tika/extractor/package-info.java +++ b/tika-core/src/main/java/org/apache/tika/extractor/package-info.java @@ -18,5 +18,5 @@ /** * Extraction of component documents. */ [email protected]("1.0.0") [email protected]("2.0.0") package org.apache.tika.extractor; diff --git a/tika-core/src/main/java/org/apache/tika/io/ByteArraySource.java b/tika-core/src/main/java/org/apache/tika/io/ByteArraySource.java index 6a2046cdf4..d2d3bbd61a 100644 --- a/tika-core/src/main/java/org/apache/tika/io/ByteArraySource.java +++ b/tika-core/src/main/java/org/apache/tika/io/ByteArraySource.java @@ -58,6 +58,10 @@ class ByteArraySource extends InputStream implements TikaInputSource { @Override public int read(byte[] b, int off, int len) { + // InputStream contract: a zero-length read returns 0, even at EOF + if (len == 0) { + return 0; + } if (position >= length) { return -1; } @@ -116,7 +120,7 @@ class ByteArraySource extends InputStream implements TikaInputSource { } @Override - public void enableRewind() { + public void enableRewind() throws IOException { // No-op: byte array is always rewindable } diff --git a/tika-core/src/main/java/org/apache/tika/io/CachingInputStream.java b/tika-core/src/main/java/org/apache/tika/io/CachingInputStream.java index b9474b1cae..c17537ba22 100644 --- a/tika-core/src/main/java/org/apache/tika/io/CachingInputStream.java +++ b/tika-core/src/main/java/org/apache/tika/io/CachingInputStream.java @@ -182,6 +182,11 @@ class CachingInputStream extends InputStream { cache.close(); } + /** For the owner to close after a spill. */ + InputStream getSource() { + return source; + } + @Override public int available() throws IOException { // Return cached bytes available from current position diff --git a/tika-core/src/main/java/org/apache/tika/io/CachingSource.java b/tika-core/src/main/java/org/apache/tika/io/CachingSource.java index 7d4c648b99..944bd54c45 100644 --- a/tika-core/src/main/java/org/apache/tika/io/CachingSource.java +++ b/tika-core/src/main/java/org/apache/tika/io/CachingSource.java @@ -43,6 +43,8 @@ class CachingSource extends InputStream implements TikaInputSource { private final TemporaryResources tmp; private final Metadata metadata; + // temp-file suffix for threshold spills, which precede any getPath(suffix) call + private final String suffix; private long length; // Passthrough mode: just a BufferedInputStream @@ -56,11 +58,16 @@ class CachingSource extends InputStream implements TikaInputSource { private Path spilledPath; private InputStream fileStream; private long filePosition; // Track position in file mode + // Retained after a spill so close() can still close it; not closed at spill + // time because an archive stream may still be in use. + private InputStream spilledSource; - CachingSource(InputStream source, TemporaryResources tmp, long length, Metadata metadata) { + CachingSource(InputStream source, TemporaryResources tmp, long length, Metadata metadata, + String suffix) { this.tmp = tmp; this.length = length; this.metadata = metadata; + this.suffix = suffix; // Start in passthrough mode this.passthroughStream = source instanceof BufferedInputStream ? (BufferedInputStream) source @@ -178,20 +185,20 @@ class CachingSource extends InputStream implements TikaInputSource { } @Override - public void enableRewind() { + public void enableRewind() throws IOException { // Already in caching or file mode - no-op if (cachingStream != null || fileStream != null) { return; } if (passthroughPosition != 0) { - throw new IllegalStateException( + throw new IOException( "Cannot enable rewind: position is " + passthroughPosition + ", must be 0. Call enableRewind() before reading."); } // Switch to caching mode - StreamCache cache = new StreamCache(tmp); + StreamCache cache = new StreamCache(tmp, suffix); cachingStream = new CachingInputStream(passthroughStream, cache); passthroughStream = null; } @@ -246,7 +253,8 @@ class CachingSource extends InputStream implements TikaInputSource { // Get current position before closing cache long currentPosition = cachingStream.getPosition(); - // Close only the cache, not the source stream (for archive support) + // close only the cache; close() releases the source later + spilledSource = cachingStream.getSource(); cachingStream.closeCacheOnly(); // Open file stream at current position @@ -256,11 +264,9 @@ class CachingSource extends InputStream implements TikaInputSource { } filePosition = currentPosition; - // Update length from file size - long fileSize = Files.size(spilledPath); - if (length == -1 || fileSize > 0) { - length = fileSize; - } + // The spooled size is ground truth, even when it is 0 and a + // Content-Length claimed otherwise + length = Files.size(spilledPath); // Update metadata if not already set if (metadata != null && @@ -289,5 +295,8 @@ class CachingSource extends InputStream implements TikaInputSource { if (passthroughStream != null) { passthroughStream.close(); } + if (spilledSource != null) { + spilledSource.close(); + } } } diff --git a/tika-core/src/main/java/org/apache/tika/io/FileSource.java b/tika-core/src/main/java/org/apache/tika/io/FileSource.java index 79163f0ab2..143bb2d562 100644 --- a/tika-core/src/main/java/org/apache/tika/io/FileSource.java +++ b/tika-core/src/main/java/org/apache/tika/io/FileSource.java @@ -120,7 +120,7 @@ class FileSource extends InputStream implements TikaInputSource { } @Override - public void enableRewind() { + public void enableRewind() throws IOException { // No-op: file is always rewindable } diff --git a/tika-core/src/main/java/org/apache/tika/io/StreamCache.java b/tika-core/src/main/java/org/apache/tika/io/StreamCache.java index d474b189aa..87a71fe931 100644 --- a/tika-core/src/main/java/org/apache/tika/io/StreamCache.java +++ b/tika-core/src/main/java/org/apache/tika/io/StreamCache.java @@ -41,18 +41,27 @@ class StreamCache implements Closeable { private int memorySize; // File storage (null until spill) + private String suffix; private Path spillFile; + // one long-lived read handle; opening per read cost ~37x on byte-at-a-time readers + private RandomAccessFile reader; private OutputStream spillOutputStream; private long totalSize; private boolean closed; StreamCache(TemporaryResources tmp) { - this(tmp, DEFAULT_MEMORY_THRESHOLD); + this(tmp, null, DEFAULT_MEMORY_THRESHOLD); + } + + /** Suffix up front: a threshold spill precedes any getPath(suffix) call (TIKA-3903). */ + StreamCache(TemporaryResources tmp, String suffix) { + this(tmp, suffix, DEFAULT_MEMORY_THRESHOLD); } - StreamCache(TemporaryResources tmp, int memoryThreshold) { + StreamCache(TemporaryResources tmp, String suffix, int memoryThreshold) { this.tmp = tmp; + this.suffix = suffix; this.memoryThreshold = memoryThreshold; this.memoryBuffer = new byte[Math.min(memoryThreshold, 8192)]; this.memorySize = 0; @@ -116,8 +125,6 @@ class StreamCache implements Closeable { memoryBuffer = newBuffer; } - private String suffix; - private void spillToFile() throws IOException { if (spillFile != null) { return; // Already spilled @@ -148,10 +155,9 @@ class StreamCache implements Closeable { return memoryBuffer[(int) position] & 0xFF; } else { flushSpillStream(); - try (RandomAccessFile raf = new RandomAccessFile(spillFile.toFile(), "r")) { - raf.seek(position); - return raf.read(); - } + RandomAccessFile raf = reader(); + raf.seek(position); + return raf.read(); } } @@ -170,10 +176,9 @@ class StreamCache implements Closeable { return available; } else { flushSpillStream(); - try (RandomAccessFile raf = new RandomAccessFile(spillFile.toFile(), "r")) { - raf.seek(position); - return raf.read(b, off, available); - } + RandomAccessFile raf = reader(); + raf.seek(position); + return raf.read(b, off, available); } } @@ -184,6 +189,13 @@ class StreamCache implements Closeable { return new CacheInputStream(offset); } + private RandomAccessFile reader() throws IOException { + if (reader == null) { + reader = new RandomAccessFile(spillFile.toFile(), "r"); + } + return reader; + } + private void flushSpillStream() throws IOException { if (spillOutputStream != null) { spillOutputStream.flush(); @@ -242,6 +254,10 @@ class StreamCache implements Closeable { spillOutputStream.close(); spillOutputStream = null; } + if (reader != null) { + reader.close(); + reader = null; + } // spillFile cleanup is handled by TemporaryResources } diff --git a/tika-core/src/main/java/org/apache/tika/io/TikaInputSource.java b/tika-core/src/main/java/org/apache/tika/io/TikaInputSource.java index 1620614f99..4c6c6e88df 100644 --- a/tika-core/src/main/java/org/apache/tika/io/TikaInputSource.java +++ b/tika-core/src/main/java/org/apache/tika/io/TikaInputSource.java @@ -64,5 +64,5 @@ interface TikaInputSource extends Closeable { * * @throws IllegalStateException if position is not 0 */ - void enableRewind(); + void enableRewind() throws IOException; } diff --git a/tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java b/tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java index 94b6db04b1..6eb1cf90d7 100644 --- a/tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java +++ b/tika-core/src/main/java/org/apache/tika/io/TikaInputStream.java @@ -47,7 +47,8 @@ import org.apache.tika.utils.StringUtils; * <ul> * <li>{@link ByteArraySource} for byte[] inputs - no caching needed</li> * <li>{@link FileSource} for Path/File inputs - direct file access</li> - * <li>{@link CachingSource} for InputStream inputs - caches bytes as read</li> + * <li>{@link CachingSource} for InputStream inputs - passthrough by default; + * caches bytes only after {@link #enableRewind()}</li> * </ul> * * @since Apache Tika 0.8 @@ -57,7 +58,7 @@ public class TikaInputStream extends TaggedInputStream { private static final int MAX_CONSECUTIVE_EOFS = 1000; private static final int BLOB_SIZE_THRESHOLD = 1024 * 1024; - protected TemporaryResources tmp; + private final TemporaryResources tmp; private long position = 0; private long mark = -1; @@ -69,14 +70,6 @@ public class TikaInputStream extends TaggedInputStream { // ========== Constructors ========== - /** - * Protected constructor for subclasses. - */ - protected TikaInputStream(InputStream stream, long length) { - super(stream); - this.tmp = null; - } - /** * Strategy-based constructor. * TikaInputSource extends InputStream, so we pass it directly to super(). @@ -104,7 +97,7 @@ public class TikaInputStream extends TaggedInputStream { return (TikaInputStream) stream; } String ext = getExtension(metadata); - TikaInputSource inputSource = new CachingSource(stream, tmp, -1, metadata); + TikaInputSource inputSource = new CachingSource(stream, tmp, -1, metadata, ext); return new TikaInputStream(inputSource, tmp, ext); } @@ -116,11 +109,11 @@ public class TikaInputStream extends TaggedInputStream { return get(stream, new TemporaryResources(), metadata); } - public static TikaInputStream get(byte[] data) throws IOException { + public static TikaInputStream get(byte[] data) { return get(data, new Metadata()); } - public static TikaInputStream get(byte[] data, Metadata metadata) throws IOException { + public static TikaInputStream get(byte[] data, Metadata metadata) { metadata.set(HttpHeaders.CONTENT_LENGTH, Integer.toString(data.length)); String ext = getExtension(metadata); TemporaryResources tmp = new TemporaryResources(); @@ -181,7 +174,7 @@ public class TikaInputStream extends TaggedInputStream { String ext = getExtension(metadata); TemporaryResources tmp = new TemporaryResources(); TikaInputSource inputSource = new CachingSource( - new BufferedInputStream(blob.getBinaryStream()), tmp, length, metadata); + new BufferedInputStream(blob.getBinaryStream()), tmp, length, metadata, ext); return new TikaInputStream(inputSource, tmp, ext); } } @@ -242,12 +235,11 @@ public class TikaInputStream extends TaggedInputStream { String ext = getExtension(metadata); TemporaryResources tmp = new TemporaryResources(); TikaInputSource inputSource = new CachingSource( - new BufferedInputStream(connection.getInputStream()), tmp, length, metadata); + new BufferedInputStream(connection.getInputStream()), tmp, length, metadata, ext); return new TikaInputStream(inputSource, tmp, ext); } - public static TikaInputStream getFromContainer(Object openContainer, long length, Metadata metadata) - throws IOException { + public static TikaInputStream getFromContainer(Object openContainer, long length, Metadata metadata) { TikaInputStream tis = TikaInputStream.get(new byte[0], metadata); tis.setOpenContainer(openContainer); tis.setLength(length); @@ -340,18 +332,20 @@ public class TikaInputStream extends TaggedInputStream { public int peek(byte[] buffer) throws IOException { int n = 0; mark(buffer.length); - - int m = read(buffer); - while (m != -1) { - n += m; - if (n < buffer.length) { - m = read(buffer, n, buffer.length - n); - } else { - m = -1; + // reset in finally: a throw mid-read must not leave the stream advanced + try { + int m = read(buffer); + while (m != -1) { + n += m; + if (n < buffer.length) { + m = read(buffer, n, buffer.length - n); + } else { + m = -1; + } } + } finally { + reset(); } - - reset(); return n; } @@ -401,6 +395,10 @@ public class TikaInputStream extends TaggedInputStream { return source != null && source.getLength() != -1; } + /** + * The stream length. For a stream-backed instance with no declared length this + * spools the entire remaining stream to a temporary file to measure it. + */ public long getLength() throws IOException { if (overrideLength >= 0) { return overrideLength; @@ -422,10 +420,6 @@ public class TikaInputStream extends TaggedInputStream { return position; } - protected void setPosition(long position) { - this.position = position; - } - private void setLength(long length) { this.overrideLength = length; } @@ -435,7 +429,10 @@ public class TikaInputStream extends TaggedInputStream { } public void removeCloseShield() { - this.closeShieldDepth--; + // floored: an unmatched remove must not cancel a later caller's shield + if (closeShieldDepth > 0) { + closeShieldDepth--; + } } public boolean isCloseShield() { @@ -477,9 +474,10 @@ public class TikaInputStream extends TaggedInputStream { * For streaming-only operations (e.g., HTML parsing), skip this call * to avoid unnecessary caching overhead. * - * @throws IllegalStateException if position is not 0 + * @throws IOException if bytes have already been read from the stream + * (position is not 0); rewind support cannot be enabled retroactively */ - public void enableRewind() { + public void enableRewind() throws IOException { TikaInputSource source = inputSource(); if (source != null) { source.enableRewind(); diff --git a/tika-core/src/main/java/org/apache/tika/io/package-info.java b/tika-core/src/main/java/org/apache/tika/io/package-info.java index 36c7274da5..3c828b965c 100644 --- a/tika-core/src/main/java/org/apache/tika/io/package-info.java +++ b/tika-core/src/main/java/org/apache/tika/io/package-info.java @@ -18,5 +18,5 @@ /** * IO utilities. */ [email protected]("1.0.0") [email protected]("2.0.0") package org.apache.tika.io; diff --git a/tika-core/src/main/java/org/apache/tika/metadata/ISO19115.java b/tika-core/src/main/java/org/apache/tika/metadata/ISO19115.java index 870e5445da..c93f0e6151 100644 --- a/tika-core/src/main/java/org/apache/tika/metadata/ISO19115.java +++ b/tika-core/src/main/java/org/apache/tika/metadata/ISO19115.java @@ -66,7 +66,7 @@ public interface ISO19115 { Property OTHER_CONSTRAINTS = Property.externalTextBag("iso19115:other-constraints"); - Property USER_CONSTRAINTS = Property.externalTextBag("iso19115:user-constraints"); + Property USE_CONSTRAINTS = Property.externalTextBag("iso19115:use-constraints"); Property GEOGRAPHIC_IDENTIFIER_CODE = Property.externalTextBag("iso19115:geographic-identifier-code"); diff --git a/tika-core/src/main/java/org/apache/tika/metadata/Metadata.java b/tika-core/src/main/java/org/apache/tika/metadata/Metadata.java index b76bbe306e..d5a9467dbf 100644 --- a/tika-core/src/main/java/org/apache/tika/metadata/Metadata.java +++ b/tika-core/src/main/java/org/apache/tika/metadata/Metadata.java @@ -101,13 +101,13 @@ public class Metadata implements Serializable { * this is skipped (with a WARN). Configurable policy belongs in a * {@link org.apache.tika.metadata.writelimiter.StandardMetadataLimiter}. */ - public static final int MAX_PREFIX_ROUTE_NAME_LENGTH = 1024; + static final int MAX_PREFIX_ROUTE_NAME_LENGTH = 1024; /** * Safety net on {@link #add(KeyPrefix, String, String)}: after this many distinct * prefix-route names in one Metadata instance, further new names are skipped (WARN once). */ - public static final int MAX_PREFIX_ROUTE_NAMES = 10_000; + static final int MAX_PREFIX_ROUTE_NAMES = 10_000; /** * Some parsers will have the date as a ISO-8601 string * already, and will set that into the Metadata object. diff --git a/tika-core/src/main/java/org/apache/tika/metadata/package-info.java b/tika-core/src/main/java/org/apache/tika/metadata/package-info.java index 02fcae3ec8..b85b43ce0d 100644 --- a/tika-core/src/main/java/org/apache/tika/metadata/package-info.java +++ b/tika-core/src/main/java/org/apache/tika/metadata/package-info.java @@ -18,5 +18,5 @@ /** * Multi-valued metadata container, and set of constant metadata fields. */ [email protected]("1.0.0") [email protected]("2.0.0") package org.apache.tika.metadata; diff --git a/tika-core/src/main/java/org/apache/tika/mime/package-info.java b/tika-core/src/main/java/org/apache/tika/mime/package-info.java index 104dc3acf9..aa45c694e2 100644 --- a/tika-core/src/main/java/org/apache/tika/mime/package-info.java +++ b/tika-core/src/main/java/org/apache/tika/mime/package-info.java @@ -18,5 +18,5 @@ /** * Media type information. */ [email protected]("1.2.0") [email protected]("2.0.0") package org.apache.tika.mime; diff --git a/tika-core/src/main/java/org/apache/tika/package-info.java b/tika-core/src/main/java/org/apache/tika/package-info.java index cf4352ddcb..5a189641a6 100644 --- a/tika-core/src/main/java/org/apache/tika/package-info.java +++ b/tika-core/src/main/java/org/apache/tika/package-info.java @@ -18,5 +18,5 @@ /** * Apache Tika. */ [email protected]("1.0.0") [email protected]("2.0.0") package org.apache.tika; diff --git a/tika-core/src/main/java/org/apache/tika/parser/ParsingIntent.java b/tika-core/src/main/java/org/apache/tika/parser/ParsingIntent.java index 60f11101d9..bdc995afe5 100644 --- a/tika-core/src/main/java/org/apache/tika/parser/ParsingIntent.java +++ b/tika-core/src/main/java/org/apache/tika/parser/ParsingIntent.java @@ -17,24 +17,17 @@ package org.apache.tika.parser; /** - * Marker class to indicate parsing intent in ParseContext. + * Parsing intent, set in the ParseContext before detection. * <p> - * When set in the ParseContext before detection, this signals to detectors - * that parsing will follow detection. Detectors can use this hint to perform - * additional preparation (like salvaging corrupted ZIP files) that would - * benefit the subsequent parse operation. + * {@link #WILL_PARSE} signals detectors that parsing will follow, so they may + * prepare durable state for the parser (e.g. salvage a corrupted ZIP into the + * open container). Set automatically by {@link AutoDetectParser}. * <p> - * This is automatically set by {@link AutoDetectParser} before calling - * the detector. + * Consumers must compare against a specific constant, never test for mere + * presence, so that new intents can be added without changing their behavior. */ -public final class ParsingIntent { +public enum ParsingIntent { - /** - * Singleton instance indicating that parsing will follow detection. - */ - public static final ParsingIntent WILL_PARSE = new ParsingIntent(); - - private ParsingIntent() { - // Private constructor for singleton - } + /** Parsing will follow detection. */ + WILL_PARSE } diff --git a/tika-core/src/main/java/org/apache/tika/parser/package-info.java b/tika-core/src/main/java/org/apache/tika/parser/package-info.java index 10df69e1e6..8142132eeb 100644 --- a/tika-core/src/main/java/org/apache/tika/parser/package-info.java +++ b/tika-core/src/main/java/org/apache/tika/parser/package-info.java @@ -18,5 +18,5 @@ /** * Tika parsers. */ [email protected]("1.0.0") [email protected]("2.0.0") package org.apache.tika.parser; diff --git a/tika-core/src/main/java/org/apache/tika/sax/BasicContentHandlerFactory.java b/tika-core/src/main/java/org/apache/tika/sax/BasicContentHandlerFactory.java index 09d7730a8d..6e787625a4 100644 --- a/tika-core/src/main/java/org/apache/tika/sax/BasicContentHandlerFactory.java +++ b/tika-core/src/main/java/org/apache/tika/sax/BasicContentHandlerFactory.java @@ -118,8 +118,9 @@ public class BasicContentHandlerFactory implements StreamingContentHandlerFactor } /** Accepted spellings, for error messages. */ + // String.join defeats compile-time constant inlining into downstream jars public static final String VALID_HANDLER_TYPE_NAMES = - "text, txt, html, xml, body, markdown, md, ignore"; + String.join(", ", "text", "txt", "html", "xml", "body", "markdown", "md", "ignore"); /** * Parses a string into a handler type. diff --git a/tika-core/src/main/java/org/apache/tika/sax/package-info.java b/tika-core/src/main/java/org/apache/tika/sax/package-info.java index 3c0b4ba48d..f2522c4e1f 100644 --- a/tika-core/src/main/java/org/apache/tika/sax/package-info.java +++ b/tika-core/src/main/java/org/apache/tika/sax/package-info.java @@ -18,5 +18,5 @@ /** * SAX utilities. */ [email protected]("1.0.0") [email protected]("2.0.0") package org.apache.tika.sax; diff --git a/tika-core/src/main/java/org/apache/tika/sax/xpath/package-info.java b/tika-core/src/main/java/org/apache/tika/sax/xpath/package-info.java index f9c1801bd6..1a12e29aa8 100644 --- a/tika-core/src/main/java/org/apache/tika/sax/xpath/package-info.java +++ b/tika-core/src/main/java/org/apache/tika/sax/xpath/package-info.java @@ -18,5 +18,5 @@ /** * XPath utilities */ [email protected]("1.0.0") [email protected]("2.0.0") package org.apache.tika.sax.xpath; diff --git a/tika-core/src/main/java/org/apache/tika/utils/package-info.java b/tika-core/src/main/java/org/apache/tika/utils/package-info.java index 04ea52e5cf..be106f7415 100644 --- a/tika-core/src/main/java/org/apache/tika/utils/package-info.java +++ b/tika-core/src/main/java/org/apache/tika/utils/package-info.java @@ -18,5 +18,5 @@ /** * Utilities. */ [email protected]("1.0.0") [email protected]("2.0.0") package org.apache.tika.utils; diff --git a/tika-core/src/main/resources/org/apache/tika/metadata/metadata-migration-3x-4x.json b/tika-core/src/main/resources/org/apache/tika/metadata/metadata-migration-3x-4x.json index dfbf078146..60fe856431 100644 --- a/tika-core/src/main/resources/org/apache/tika/metadata/metadata-migration-3x-4x.json +++ b/tika-core/src/main/resources/org/apache/tika/metadata/metadata-migration-3x-4x.json @@ -71,7 +71,7 @@ {"v3":"TransferOptionsOnlineProtocol ","v4":"iso19115:transfer-options-online-protocol"}, {"v3":"Upper Left","v4":"gdal:upper-left"}, {"v3":"Upper Right","v4":"gdal:upper-right"}, -{"v3":"UserConstraints ","v4":"iso19115:user-constraints"}, +{"v3":"UserConstraints ","v4":"iso19115:use-constraints"}, {"v3":"X-TIKA:EXCEPTION:container_exception","v4":"tk:exception:container-exception"}, {"v3":"X-TIKA:EXCEPTION:embedded_bytes_exception","v4":"tk:exception:embedded-bytes-exception"}, {"v3":"X-TIKA:EXCEPTION:embedded_exception","v4":"tk:exception:embedded-exception"}, diff --git a/tika-core/src/test/java/org/apache/tika/TikaDetectStreamOwnershipTest.java b/tika-core/src/test/java/org/apache/tika/TikaDetectStreamOwnershipTest.java new file mode 100644 index 0000000000..0b1cb319dd --- /dev/null +++ b/tika-core/src/test/java/org/apache/tika/TikaDetectStreamOwnershipTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; + +/** {@link Tika#detect} peeks: it must dispose only what it spooled itself. */ +public class TikaDetectStreamOwnershipTest { + + private static final byte[] DATA = "hello world\n".getBytes(StandardCharsets.UTF_8); + + private static class CloseCountingInputStream extends ByteArrayInputStream { + int closes = 0; + + CloseCountingInputStream(byte[] buf) { + super(buf); + } + + @Override + public void close() throws IOException { + closes++; + super.close(); + } + } + + @Test + public void testDetectDoesNotCloseCallersStream() throws Exception { + Tika tika = new Tika(); + CloseCountingInputStream stream = new CloseCountingInputStream(DATA); + tika.detect(stream, new Metadata()); + assertEquals(0, stream.closes, + "detect() must not close a stream it was handed; the javadoc promises this"); + } + + /** get() returns a caller-supplied TikaInputStream as-is, so disposing it here + * would delete the caller's temp file. */ + @Test + public void testDetectDoesNotDisposeCallerOwnedTikaInputStream() throws Exception { + Tika tika = new Tika(); + try (TikaInputStream tis = TikaInputStream.get(DATA)) { + Path spooled = tis.getPath(); + assertTrue(Files.exists(spooled), "precondition: caller spooled a temp file"); + + tika.detect(tis, new Metadata()); + + assertTrue(Files.exists(spooled), + "detect() must not dispose a caller-owned TikaInputStream's temp file"); + assertTrue(tis.getPath().toFile().exists(), "the stream must still be usable"); + } + } +} diff --git a/tika-core/src/test/java/org/apache/tika/io/TikaInputStreamTest.java b/tika-core/src/test/java/org/apache/tika/io/TikaInputStreamTest.java index 2f8915b5f1..ac86ec3b05 100644 --- a/tika-core/src/test/java/org/apache/tika/io/TikaInputStreamTest.java +++ b/tika-core/src/test/java/org/apache/tika/io/TikaInputStreamTest.java @@ -684,7 +684,7 @@ public class TikaInputStreamTest { try (TemporaryResources tmp = new TemporaryResources()) { CachingSource source = new CachingSource( - new ByteArrayInputStream(data), tmp, -1, metadata); + new ByteArrayInputStream(data), tmp, -1, metadata, null); source.enableRewind(); // Enable caching for spill support // Read all data @@ -720,7 +720,7 @@ public class TikaInputStreamTest { try (TemporaryResources tmp = new TemporaryResources()) { CachingSource source = new CachingSource( - new ByteArrayInputStream(data), tmp, -1, metadata); + new ByteArrayInputStream(data), tmp, -1, metadata, null); source.enableRewind(); // Enable caching for seek/spill support // Read and spill @@ -741,7 +741,7 @@ public class TikaInputStreamTest { try (TemporaryResources tmp = new TemporaryResources()) { CachingSource source = new CachingSource( - new ByteArrayInputStream(data), tmp, -1, null); + new ByteArrayInputStream(data), tmp, -1, null, null); source.enableRewind(); // Enable caching for seek support // Read first 5 bytes @@ -767,7 +767,7 @@ public class TikaInputStreamTest { try (TemporaryResources tmp = new TemporaryResources()) { CachingSource source = new CachingSource( - new ByteArrayInputStream(data), tmp, -1, null); + new ByteArrayInputStream(data), tmp, -1, null, null); source.enableRewind(); // Enable caching for spill/seek support // Read first 5 bytes @@ -864,7 +864,7 @@ public class TikaInputStreamTest { tis.read(); // Read one byte, position is now 1 assertEquals(1, tis.getPosition()); - assertThrows(IllegalStateException.class, tis::enableRewind, + assertThrows(IOException.class, tis::enableRewind, "enableRewind() should throw when position != 0"); } } @@ -996,4 +996,55 @@ public class TikaInputStreamTest { private static String str(byte[] b) { return new String(b, UTF_8); } + + private static class CloseCountingInputStream extends ByteArrayInputStream { + int closes = 0; + + CloseCountingInputStream(byte[] buf) { + super(buf); + } + + @Override + public void close() throws IOException { + closes++; + super.close(); + } + } + + /** + * Spill must not close the source (an archive stream may still be in use), + * but close() must reach it exactly once. + */ + @Test + public void testSourceClosedOnceAfterSpill() throws Exception { + CloseCountingInputStream source = new CloseCountingInputStream(new byte[64]); + Path spilled; + try (TikaInputStream tis = TikaInputStream.get(source)) { + spilled = tis.getPath(); + assertEquals(0, source.closes, "source must stay open at spill time"); + } + assertEquals(1, source.closes, "close() must close the spilled source exactly once"); + assertFalse(Files.exists(spilled), "temp file must be deleted on close"); + } + + /** TIKA-3903: the suffix must survive a spill triggered by the memory threshold. */ + @Test + public void testSuffixSurvivesThresholdSpill() throws Exception { + Metadata metadata = new Metadata(); + metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "scan.png"); + // one byte past the StreamCache memory threshold + byte[] data = new byte[1024 * 1024 + 1]; + try (TikaInputStream tis = + TikaInputStream.get(new ByteArrayInputStream(data), metadata)) { + tis.enableRewind(); + // drain past the threshold so the cache spills before getPath() + byte[] buffer = new byte[8192]; + while (tis.read(buffer) != -1) { + } + tis.rewind(); + assertTrue(tis.getPath().getFileName().toString().endsWith(".png"), + "threshold spill must keep the original suffix, got: " + + tis.getPath().getFileName()); + } + } } diff --git a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-key-fields.json b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-key-fields.json index 7c79e191e7..3392c0e3be 100644 --- a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-key-fields.json +++ b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-key-fields.json @@ -231,7 +231,7 @@ {"class":"org.apache.tika.metadata.ISO19115","field":"TRANSFER_OPTIONS_ONLINE_NAME","key":"iso19115:transfer-options-online-name"}, {"class":"org.apache.tika.metadata.ISO19115","field":"TRANSFER_OPTIONS_ONLINE_PROFILE","key":"iso19115:transfer-options-online-profile"}, {"class":"org.apache.tika.metadata.ISO19115","field":"TRANSFER_OPTIONS_ONLINE_PROTOCOL","key":"iso19115:transfer-options-online-protocol"}, - {"class":"org.apache.tika.metadata.ISO19115","field":"USER_CONSTRAINTS","key":"iso19115:user-constraints"}, + {"class":"org.apache.tika.metadata.ISO19115","field":"USE_CONSTRAINTS","key":"iso19115:use-constraints"}, {"class":"org.apache.tika.metadata.MAPI","field":"ATTACH_CONTENT_ID","key":"mapi:attach:content-id"}, {"class":"org.apache.tika.metadata.MAPI","field":"ATTACH_CONTENT_LOCATION","key":"mapi:attach:content-location"}, {"class":"org.apache.tika.metadata.MAPI","field":"ATTACH_DISPLAY_NAME","key":"mapi:attach:display-name"}, diff --git a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json index b0a7aea762..817b002655 100644 --- a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json +++ b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json @@ -243,7 +243,7 @@ {"key":"iso19115:transfer-options-online-name","namespace":"iso19115","valueType":"TEXT","cardinality":"BAG","module":"tika-core"}, {"key":"iso19115:transfer-options-online-profile","namespace":"iso19115","valueType":"TEXT","cardinality":"BAG","module":"tika-core"}, {"key":"iso19115:transfer-options-online-protocol","namespace":"iso19115","valueType":"TEXT","cardinality":"BAG","module":"tika-core"}, - {"key":"iso19115:user-constraints","namespace":"iso19115","valueType":"TEXT","cardinality":"BAG","module":"tika-core"}, + {"key":"iso19115:use-constraints","namespace":"iso19115","valueType":"TEXT","cardinality":"BAG","module":"tika-core"}, {"key":"iwork:build-version-history","namespace":"iwork","valueType":"TEXT","cardinality":"BAG","module":"tika-parser-apple-module"}, {"key":"iwork:document-id","namespace":"iwork","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-parser-apple-module"}, {"key":"keynote:slides-height","namespace":"keynote","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-parser-apple-module"}, diff --git a/tika-metadata-schema/src/test/resources/org/apache/tika/metadata/migration-overlay.tsv b/tika-metadata-schema/src/test/resources/org/apache/tika/metadata/migration-overlay.tsv index fbd2496652..8288dbedaa 100644 --- a/tika-metadata-schema/src/test/resources/org/apache/tika/metadata/migration-overlay.tsv +++ b/tika-metadata-schema/src/test/resources/org/apache/tika/metadata/migration-overlay.tsv @@ -197,7 +197,7 @@ IdentificationInfoTopicCategory--> iso19115:identification-info-topic-category ThesaurusNameDate iso19115:thesaurus-name-date AccessContraints iso19115:access-constraints OtherConstraints iso19115:other-constraints -UserConstraints iso19115:user-constraints +UserConstraints iso19115:use-constraints GeographicIdentifierCode iso19115:geographic-identifier-code GeographicIdentifierAuthorityTitle iso19115:geographic-identifier-authority-title GeographicIdentifierAuthorityAlternativeTitle iso19115:geographic-identifier-authority-alternative-title diff --git a/tika-parent/pom.xml b/tika-parent/pom.xml index 9efe070db6..6d8ad0e9fa 100644 --- a/tika-parent/pom.xml +++ b/tika-parent/pom.xml @@ -1724,7 +1724,7 @@ possible'). tika-grpc is excluded: its protobuf generated-sources and compile deps are not on the aggregate classpath. NOTE: keep this list in sync when modules are added or removed (see TIKA-4318). --> - <sourcepath>tika-annotation-processor/src/main/java;tika-app/src/main/java;tika-bundles/tika-bundle-standard/src/main/java;tika-core/src/main/java;tika-detectors/tika-detector-magika/src/main/java;tika-detectors/tika-detector-siegfried/src/main/java;tika-encoding-detectors/tika-encoding-detector-html/src/main/java;tika-encoding-detectors/tika-encoding-detector-icu4j/src/main/java;tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java;tika-encoding-detectors [...] + <sourcepath>tika-annotation-processor/src/main/java;tika-app/src/main/java;tika-bundles/tika-bundle-standard/src/main/java;tika-core/src/main/java;tika-detectors/tika-detector-magika/src/main/java;tika-detectors/tika-detector-siegfried/src/main/java;tika-encoding-detectors/tika-encoding-detector-html/src/main/java;tika-encoding-detectors/tika-encoding-detector-icu4j/src/main/java;tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java;tika-encoding-detectors [...] <subpackages>org.apache.tika</subpackages> </configuration> </plugin> diff --git a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/geoinfo/GeographicInformationParser.java b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/geoinfo/GeographicInformationParser.java index c006475746..c0c858518f 100644 --- a/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/geoinfo/GeographicInformationParser.java +++ b/tika-parsers/tika-parsers-extended/tika-parser-scientific-module/src/main/java/org/apache/tika/parser/geoinfo/GeographicInformationParser.java @@ -357,7 +357,7 @@ public class GeographicInformationParser implements Parser { metadata.add(ISO19115.OTHER_CONSTRAINTS, s.toString()); } for (Restriction r : c.getUseConstraints()) { - metadata.add(ISO19115.USER_CONSTRAINTS, r.name()); + metadata.add(ISO19115.USE_CONSTRAINTS, r.name()); } } diff --git a/tika-parsers/tika-parsers-ml/tika-vlm/pom.xml b/tika-parsers/tika-parsers-ml/tika-vlm/pom.xml index 409e7ebbc0..2824428b1b 100644 --- a/tika-parsers/tika-parsers-ml/tika-vlm/pom.xml +++ b/tika-parsers/tika-parsers-ml/tika-vlm/pom.xml @@ -75,6 +75,13 @@ <version>${project.version}</version> <scope>test</scope> </dependency> + <!-- vlm-pdf-parsing.json excludes 'pdf-parser', which must resolve (TIKA-3268) --> + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>tika-parser-pdf-module</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> </dependencies> <build> diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/RecursiveParserWrapperTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/RecursiveParserWrapperTest.java index 6b9d18792c..fb4d30525d 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/RecursiveParserWrapperTest.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/RecursiveParserWrapperTest.java @@ -455,9 +455,7 @@ public class RecursiveParserWrapperTest extends TikaTest { @Test public void testStreamClosedAfterSpill() throws Exception { - // When TikaInputStream spills to a temp file (via getPath()/getFile()), - // the source stream should be closed promptly since all bytes have been - // consumed and cached - there's no reason to keep it open. + // Spill must not close the source early; close() must reach it exactly once. ParseContext context = new ParseContext(); Metadata metadata = new Metadata(); RecursiveParserWrapper wrapper = new RecursiveParserWrapper(AUTO_DETECT_PARSER, true); @@ -472,10 +470,11 @@ public class RecursiveParserWrapperTest extends TikaTest { TikaInputStream tis = TikaInputStream.get(stream); tis.setCloseShield(); wrapper.parse(tis, handler, metadata, context); - // Source stream should not be closed after spilling to file assertEquals(0, stream.counter); tis.removeCloseShield(); tis.close(); + // close() must reach the source, spill or no spill + assertEquals(1, stream.counter); } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java index e1b81fce41..c61f33cc45 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pdf-module/src/main/java/org/apache/tika/parser/pdf/PDFParser.java @@ -253,16 +253,15 @@ public class PDFParser implements Parser, RenderingParser { } private PDFParserConfig getConfig(ParseContext parseContext) throws TikaException, IOException { - // ParseContextConfig.getConfig() handles: - // 1. Check for PDFParserConfig already in ParseContext (fast path for embedded docs) - // 2. Check jsonConfigs for "pdf-parser" and deserialize if present - // 3. Set deserialized config in ParseContext for future lookups - // 4. Return defaultConfig if no runtime config found - return ParseContextConfig.getConfig( + PDFParserConfig config = ParseContextConfig.getConfig( parseContext, "pdf-parser", PDFParserConfig.class, defaultConfig); + // publish class-keyed for collaborators that read parseContext.get(PDFParserConfig.class), + // e.g. PDFBoxRenderer; safe because pdf-parser is this class's sole owner + parseContext.set(PDFParserConfig.class, config); + return config; } private void checkEncryptedPayload(PDDocument pdfDocument, diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/CompressorParser.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/CompressorParser.java index 004245b4b3..7aa4200ca3 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/CompressorParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/CompressorParser.java @@ -229,6 +229,7 @@ public class CompressorParser implements Parser { tis.setCloseShield(); CompressorInputStream cis; + boolean detected = false; try { CompressorParserOptions options = context.get(CompressorParserOptions.class, @@ -272,17 +273,18 @@ public class CompressorParser implements Parser { metadata.set(CONTENT_TYPE, type.toString()); } } + detected = true; } catch (CompressorException e) { - tis.removeCloseShield(); if (e.getCause() instanceof MemoryLimitException) { throw new TikaMemoryLimitException(e.getMessage()); } throw new TikaException("Unable to uncompress document stream", e); - } catch (IOException e) { - //the pack200 workaround (getPath()/Files.newInputStream) can throw IOException; - //make sure the close shield is removed before propagating - tis.removeCloseShield(); - throw e; + } finally { + // covers unchecked escapes too (e.g. InaccessibleObjectException from the + // pack200 reflection), which used to leave the shield stuck on + if (!detected) { + tis.removeCloseShield(); + } } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-zip-commons/src/main/java/org/apache/tika/detect/zip/DefaultZipContainerDetector.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-zip-commons/src/main/java/org/apache/tika/detect/zip/DefaultZipContainerDetector.java index 9516f7b8ae..32eaff2fdc 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-zip-commons/src/main/java/org/apache/tika/detect/zip/DefaultZipContainerDetector.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-zip-commons/src/main/java/org/apache/tika/detect/zip/DefaultZipContainerDetector.java @@ -242,7 +242,7 @@ public class DefaultZipContainerDetector implements Detector { metadata.set(Zip.DETECTOR_ZIPFILE_OPENED, false); // If parsing will follow, try salvaging to prepare ZipFile for parser reuse - if (parseContext.get(ParsingIntent.class) != null) { + if (parseContext.get(ParsingIntent.class) == ParsingIntent.WILL_PARSE) { zip = ZipSalvager.tryToOpenZipFile(tis, metadata); if (zip != null && LOG.isDebugEnabled()) { LOG.debug("Successfully salvaged ZIP for parsing"); diff --git a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java b/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java index 174d940244..e6b7b9db03 100644 --- a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java +++ b/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java @@ -350,9 +350,7 @@ public class TikaAsyncCLI { if (asyncConfig == null) { return; } - if (asyncConfig.getHandlerType() == BasicContentHandlerFactory.HANDLER_TYPE.TEXT) { - return; - } + // no TEXT early-return: the default handler is MARKDOWN, not TEXT ContentHandlerFactory factory = new BasicContentHandlerFactory(asyncConfig.getHandlerType(), -1); t.getParseContext().set(ContentHandlerFactory.class, factory); } diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/UnpackExtractor.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/UnpackExtractor.java index b90e8af46c..daf4755fc9 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/UnpackExtractor.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/UnpackExtractor.java @@ -101,8 +101,8 @@ public class UnpackExtractor extends ParsingEmbeddedDocumentExtractor { // Use the delegate parser to parse this entry try { - UnpackHandler bytesHandler = context.get(UnpackHandler.class); tis.setCloseShield(); + UnpackHandler bytesHandler = context.get(UnpackHandler.class); if (bytesHandler != null) { parseWithBytes(tis, handler, metadata, context); } else { @@ -139,6 +139,9 @@ public class UnpackExtractor extends ParsingEmbeddedDocumentExtractor { translated = Files.createTempFile("tika-tmp-", ".bin"); try (OutputStream os = Files.newOutputStream(translated)) { EMBEDDED_STREAM_TRANSLATOR.translate(tis, metadata, os); + } finally { + // translate() drains to EOF; without this the parse below sees an empty stream + tis.rewind(); } } parse(tis, handler, metadata, context); diff --git a/tika-pipes/tika-pipes-plugins/pom.xml b/tika-pipes/tika-pipes-plugins/pom.xml index 5f6b34050c..6172faa8d4 100644 --- a/tika-pipes/tika-pipes-plugins/pom.xml +++ b/tika-pipes/tika-pipes-plugins/pom.xml @@ -56,10 +56,12 @@ <!-- !!! VERY IMPORTANT --> <scope>provided</scope> </dependency> + <!-- provided: bundling the plugin/host boundary yields duplicate Class objects --> <dependency> <groupId>org.apache.tika</groupId> <artifactId>tika-pipes-api</artifactId> <version>${project.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>${project.groupId}</groupId> @@ -73,6 +75,45 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>org.apache.tika</groupId> + <artifactId>tika-serialization</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <!-- parent-loaded: on every host via tika-pipes-core; bundling duplicates PipesIteratorBase --> + <dependency> + <groupId>org.apache.tika</groupId> + <artifactId>tika-pipes-iterator-commons</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <!-- logging is host-supplied --> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>jcl-over-slf4j</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-slf4j2-impl</artifactId> + <scope>provided</scope> + </dependency> <dependency> <groupId>org.apache.tika</groupId> <artifactId>tika-core</artifactId> diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-atlassian-jwt/pom.xml b/tika-pipes/tika-pipes-plugins/tika-pipes-atlassian-jwt/pom.xml index a2808232c5..70f3fc6b1f 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-atlassian-jwt/pom.xml +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-atlassian-jwt/pom.xml @@ -30,8 +30,6 @@ <properties> <!-- Never include the core artifacts in your plugin lib directory. If you do, it will cause the classloading to get messed up when finding your plugins. --> - <plugin.excluded.artifactIds>tika-core,tika-pipes-api,tika-serialization,tika-plugins-core</plugin.excluded.artifactIds> - <plugin.excluded.groupIds>org.apache.logging.log4j,org.slf4j</plugin.excluded.groupIds> </properties> <dependencies> @@ -108,9 +106,7 @@ </goals> <configuration> <outputDirectory>${project.build.directory}/lib</outputDirectory> - <includeScope>compile</includeScope> - <excludeArtifactIds>${plugin.excluded.artifactIds}</excludeArtifactIds> - <excludeGroupIds>${plugin.excluded.groupIds}</excludeGroupIds> + <includeScope>runtime</includeScope> </configuration> </execution> </executions> diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-atlassian-jwt/src/main/java/org/apache/tika/pipes/fetcher/atlassianjwt/AtlassianJwtFetcher.java b/tika-pipes/tika-pipes-plugins/tika-pipes-atlassian-jwt/src/main/java/org/apache/tika/pipes/fetcher/atlassianjwt/AtlassianJwtFetcher.java index 45cbce9391..bc29143eaa 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-atlassian-jwt/src/main/java/org/apache/tika/pipes/fetcher/atlassianjwt/AtlassianJwtFetcher.java +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-atlassian-jwt/src/main/java/org/apache/tika/pipes/fetcher/atlassianjwt/AtlassianJwtFetcher.java @@ -27,6 +27,7 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.security.NoSuchAlgorithmException; import java.util.List; import java.util.Timer; @@ -237,9 +238,12 @@ public class AtlassianJwtFetcher extends AbstractTikaExtension implements Fetche private TikaInputStream spool(InputStream content, Metadata metadata) throws IOException { long start = System.currentTimeMillis(); TemporaryResources tmp = new TemporaryResources(); + boolean handedOff = false; + try { Path tmpFile = tmp.createTempFile(metadata); if (config.getMaxSpoolSize() < 0) { - Files.copy(content, tmpFile); + // createTempFile already created the file + Files.copy(content, tmpFile, StandardCopyOption.REPLACE_EXISTING); } else { try (OutputStream os = Files.newOutputStream(tmpFile)) { long totalRead = IOUtils.copyLarge(content, os, 0, config.getMaxSpoolSize()); @@ -250,7 +254,15 @@ public class AtlassianJwtFetcher extends AbstractTikaExtension implements Fetche } long elapsed = System.currentTimeMillis() - start; LOG.debug("took {} ms to copy to local tmp file", elapsed); - return TikaInputStream.get(tmpFile); + TikaInputStream tis = TikaInputStream.get(tmpFile, metadata, tmp); + handedOff = true; + return tis; + } finally { + // a failed copy must not orphan the temp file + if (!handedOff) { + tmp.close(); + } + } } private void updateMetadata(String url, HttpResponse response, HttpClientContext context, diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-az-blob/pom.xml b/tika-pipes/tika-pipes-plugins/tika-pipes-az-blob/pom.xml index 72e22d0eeb..a869844670 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-az-blob/pom.xml +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-az-blob/pom.xml @@ -32,8 +32,6 @@ <properties> <!-- You want to make sure you don't load competing dependencies in your plugin or your classloading is going to get messed up when finding your plugins. --> - <plugin.excluded.artifactIds>tika-core,tika-pipes-api,tika-serialization,tika-plugins-core</plugin.excluded.artifactIds> - <plugin.excluded.groupIds>org.apache.logging.log4j,org.slf4j</plugin.excluded.groupIds> </properties> <dependencies> @@ -43,11 +41,6 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> - <dependency> - <groupId>${project.groupId}</groupId> - <artifactId>tika-pipes-iterator-commons</artifactId> - <version>${project.version}</version> - </dependency> <dependency> <groupId>${project.groupId}</groupId> <artifactId>tika-serialization</artifactId> @@ -90,8 +83,6 @@ <configuration> <outputDirectory>${project.build.directory}/lib</outputDirectory> <includeScope>runtime</includeScope> - <excludeArtifactIds>${plugin.excluded.artifactIds}</excludeArtifactIds> - <excludeGroupIds>${plugin.excluded.groupIds}</excludeGroupIds> </configuration> </execution> </executions> diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-csv/pom.xml b/tika-pipes/tika-pipes-plugins/tika-pipes-csv/pom.xml index dc0ae87c11..a01fb2baa5 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-csv/pom.xml +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-csv/pom.xml @@ -30,16 +30,9 @@ <name>Apache Tika Pipes CSV</name> <url>https://tika.apache.org/</url> <properties> - <plugin.excluded.artifactIds>tika-core,tika-pipes-api,tika-serialization,tika-plugins-core,tika-pipes-iterator-commons</plugin.excluded.artifactIds> - <plugin.excluded.groupIds>org.apache.logging.log4j,org.slf4j</plugin.excluded.groupIds> </properties> <dependencies> - <dependency> - <groupId>${project.groupId}</groupId> - <artifactId>tika-pipes-iterator-commons</artifactId> - <version>${project.version}</version> - </dependency> <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-csv</artifactId> @@ -70,8 +63,6 @@ <configuration> <outputDirectory>target/lib</outputDirectory> <includeScope>runtime</includeScope> - <excludeArtifactIds>${plugin.excluded.artifactIds}</excludeArtifactIds> - <excludeGroupIds>${plugin.excluded.groupIds}</excludeGroupIds> </configuration> </execution> </executions> diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-es/pom.xml b/tika-pipes/tika-pipes-plugins/tika-pipes-es/pom.xml index 1170e5e970..58362fd936 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-es/pom.xml +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-es/pom.xml @@ -35,8 +35,6 @@ <properties> <!-- Never include the core artifacts in your plugin lib directory. If you do, it will cause the classloading to get messed up when finding your plugins. --> - <plugin.excluded.artifactIds>tika-core,tika-pipes-api,tika-serialization,tika-plugins-core</plugin.excluded.artifactIds> - <plugin.excluded.groupIds>org.apache.logging.log4j,org.slf4j</plugin.excluded.groupIds> </properties> <dependencies> @@ -81,8 +79,6 @@ <configuration> <outputDirectory>${project.build.directory}/lib</outputDirectory> <includeScope>runtime</includeScope> - <excludeArtifactIds>${plugin.excluded.artifactIds}</excludeArtifactIds> - <excludeGroupIds>${plugin.excluded.groupIds}</excludeGroupIds> </configuration> </execution> </executions> diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-file-system/pom.xml b/tika-pipes/tika-pipes-plugins/tika-pipes-file-system/pom.xml index 5459d99c1e..b14dddb83e 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-file-system/pom.xml +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-file-system/pom.xml @@ -30,15 +30,8 @@ <properties> <!-- Never include the core artifacts in your plugin lib directory. If you do, it will cause the classloading to get messed up when finding your plugins. --> - <plugin.excluded.artifactIds>tika-core,tika-pipes-api,tika-serialization,tika-plugins-core</plugin.excluded.artifactIds> - <plugin.excluded.groupIds>org.apache.logging.log4j,org.slf4j</plugin.excluded.groupIds> </properties> <dependencies> - <dependency> - <groupId>${project.groupId}</groupId> - <artifactId>tika-pipes-iterator-commons</artifactId> - <version>${project.version}</version> - </dependency> <dependency> <groupId>${project.groupId}</groupId> <artifactId>tika-pipes-api</artifactId> @@ -94,8 +87,6 @@ <configuration> <outputDirectory>${project.build.directory}/lib</outputDirectory> <includeScope>runtime</includeScope> - <excludeArtifactIds>${plugin.excluded.artifactIds}</excludeArtifactIds> - <excludeGroupIds>${plugin.excluded.groupIds}</excludeGroupIds> </configuration> </execution> </executions> diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/pom.xml b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/pom.xml index 2d569aa8bb..d17c56e97a 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/pom.xml +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-gcs/pom.xml @@ -32,8 +32,6 @@ <properties> <!-- You want to make sure you don't load competing dependencies in your plugin or your classloading is going to get messed up when finding your plugins. --> - <plugin.excluded.artifactIds>tika-core,tika-pipes-api,tika-serialization,tika-plugins-core</plugin.excluded.artifactIds> - <plugin.excluded.groupIds>org.apache.logging.log4j,org.slf4j</plugin.excluded.groupIds> </properties> <dependencies> @@ -43,11 +41,6 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> - <dependency> - <groupId>${project.groupId}</groupId> - <artifactId>tika-pipes-iterator-commons</artifactId> - <version>${project.version}</version> - </dependency> <dependency> <groupId>${project.groupId}</groupId> <artifactId>tika-serialization</artifactId> @@ -86,8 +79,6 @@ <configuration> <outputDirectory>${project.build.directory}/lib</outputDirectory> <includeScope>runtime</includeScope> - <excludeArtifactIds>${plugin.excluded.artifactIds}</excludeArtifactIds> - <excludeGroupIds>${plugin.excluded.groupIds}</excludeGroupIds> </configuration> </execution> </executions> diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-google-drive/pom.xml b/tika-pipes/tika-pipes-plugins/tika-pipes-google-drive/pom.xml index 05488ecb39..ac0a5f1517 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-google-drive/pom.xml +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-google-drive/pom.xml @@ -28,8 +28,6 @@ <artifactId>tika-pipes-google-drive</artifactId> <name>Apache Tika Pipes Google Drive</name> <properties> - <plugin.excluded.artifactIds>tika-core,tika-pipes-api,tika-serialization,tika-plugins-core</plugin.excluded.artifactIds> - <plugin.excluded.groupIds>org.apache.logging.log4j,org.slf4j</plugin.excluded.groupIds> <google-api-services-drive.version>v3-rev20260720-2.0.0</google-api-services-drive.version> <google-auth-library-oauth2-http.version>1.50.0</google-auth-library-oauth2-http.version> <google-api-client.version>2.9.0</google-api-client.version> @@ -154,9 +152,7 @@ </goals> <configuration> <outputDirectory>${project.build.directory}/lib</outputDirectory> - <includeScope>compile</includeScope> - <excludeArtifactIds>${plugin.excluded.artifactIds}</excludeArtifactIds> - <excludeGroupIds>${plugin.excluded.groupIds}</excludeGroupIds> + <includeScope>runtime</includeScope> </configuration> </execution> </executions> diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-google-drive/src/main/java/org/apache/tika/pipes/fetcher/googledrive/GoogleDriveFetcher.java b/tika-pipes/tika-pipes-plugins/tika-pipes-google-drive/src/main/java/org/apache/tika/pipes/fetcher/googledrive/GoogleDriveFetcher.java index 57b340a32f..7f0e13918e 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-google-drive/src/main/java/org/apache/tika/pipes/fetcher/googledrive/GoogleDriveFetcher.java +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-google-drive/src/main/java/org/apache/tika/pipes/fetcher/googledrive/GoogleDriveFetcher.java @@ -106,7 +106,8 @@ public class GoogleDriveFetcher extends AbstractTikaExtension implements Fetcher TemporaryResources tmp = new TemporaryResources(); Path tmpPath = tmp.createTempFile(metadata); FileUtils.copyInputStreamToFile(is, tmpPath.toFile()); - return TikaInputStream.get(tmpPath); + // the 3-arg form registers tmp for cleanup + return TikaInputStream.get(tmpPath, metadata, tmp); } return TikaInputStream.get(is); diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-http/pom.xml b/tika-pipes/tika-pipes-plugins/tika-pipes-http/pom.xml index 314e2b5567..7e6daf2f33 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-http/pom.xml +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-http/pom.xml @@ -30,8 +30,6 @@ <properties> <!-- Never include the core artifacts in your plugin lib directory. If you do, it will cause the classloading to get messed up when finding your plugins. --> - <plugin.excluded.artifactIds>tika-core,tika-pipes-api,tika-serialization,tika-plugins-core</plugin.excluded.artifactIds> - <plugin.excluded.groupIds>org.apache.logging.log4j,org.slf4j</plugin.excluded.groupIds> </properties> <dependencies> @@ -134,8 +132,6 @@ <configuration> <outputDirectory>target/lib</outputDirectory> <includeScope>runtime</includeScope> - <excludeArtifactIds>${plugin.excluded.artifactIds}</excludeArtifactIds> - <excludeGroupIds>${plugin.excluded.groupIds}</excludeGroupIds> </configuration> </execution> </executions> diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java b/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java index 3d0c469327..21f43a7048 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcher.java @@ -342,6 +342,8 @@ public class HttpFetcher extends AbstractTikaExtension implements Fetcher, Range private TikaInputStream spool(InputStream content, Metadata metadata) throws IOException { long start = System.currentTimeMillis(); TemporaryResources tmp = new TemporaryResources(); + boolean handedOff = false; + try { Path tmpFile = tmp.createTempFile(metadata); if (httpFetcherConfig.getMaxSpoolSize() < 0) { Files.copy(content, tmpFile, StandardCopyOption.REPLACE_EXISTING); @@ -355,7 +357,15 @@ public class HttpFetcher extends AbstractTikaExtension implements Fetcher, Range } long elapsed = System.currentTimeMillis() - start; LOG.debug("took {} ms to copy to local tmp file", elapsed); - return TikaInputStream.get(tmpFile, metadata, tmp); + TikaInputStream tis = TikaInputStream.get(tmpFile, metadata, tmp); + handedOff = true; + return tis; + } finally { + // a failed copy must not orphan the temp file + if (!handedOff) { + tmp.close(); + } + } } private void updateMetadata(String url, HttpResponse response, HttpClientContext context, Metadata metadata) { diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/config/HttpFetcherConfig.java b/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/config/HttpFetcherConfig.java index 27320af480..68efc2f69f 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/config/HttpFetcherConfig.java +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-http/src/main/java/org/apache/tika/pipes/fetcher/http/config/HttpFetcherConfig.java @@ -63,7 +63,8 @@ public class HttpFetcherConfig { private String userAgent; private String jwtIssuer; private String jwtSubject; - private int jwtExpiresInSeconds; + // 0 would mint already-expired tokens + private int jwtExpiresInSeconds = 3600; private String jwtSecret; private String jwtPrivateKeyBase64; diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-jdbc/pom.xml b/tika-pipes/tika-pipes-plugins/tika-pipes-jdbc/pom.xml index 668c6a86e3..d9a28729f2 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-jdbc/pom.xml +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-jdbc/pom.xml @@ -32,8 +32,6 @@ <properties> <!-- You want to make sure you don't load competing dependencies in your plugin or your classloading is going to get messed up when finding your plugins. --> - <plugin.excluded.artifactIds>tika-core,tika-pipes-api,tika-serialization,tika-plugins-core</plugin.excluded.artifactIds> - <plugin.excluded.groupIds>org.apache.logging.log4j,org.slf4j</plugin.excluded.groupIds> </properties> <dependencies> @@ -43,11 +41,6 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> - <dependency> - <groupId>${project.groupId}</groupId> - <artifactId>tika-pipes-iterator-commons</artifactId> - <version>${project.version}</version> - </dependency> <dependency> <groupId>${project.groupId}</groupId> <artifactId>tika-pipes-reporter-commons</artifactId> @@ -86,8 +79,6 @@ <configuration> <outputDirectory>${project.build.directory}/lib</outputDirectory> <includeScope>runtime</includeScope> - <excludeArtifactIds>${plugin.excluded.artifactIds}</excludeArtifactIds> - <excludeGroupIds>${plugin.excluded.groupIds}</excludeGroupIds> </configuration> </execution> </executions> diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-jdbc/src/main/java/org/apache/tika/pipes/reporter/jdbc/JDBCPipesReporter.java b/tika-pipes/tika-pipes-plugins/tika-pipes-jdbc/src/main/java/org/apache/tika/pipes/reporter/jdbc/JDBCPipesReporter.java index f3239b4461..7fa67f90ac 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-jdbc/src/main/java/org/apache/tika/pipes/reporter/jdbc/JDBCPipesReporter.java +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-jdbc/src/main/java/org/apache/tika/pipes/reporter/jdbc/JDBCPipesReporter.java @@ -95,6 +95,8 @@ public class JDBCPipesReporter extends PipesReporterBase { } if (config.reportSql() == null || config.reportSql().isBlank()) { reportSql = "insert into " + config.tableName() + " (id, status, timestamp) values (?,?,?)"; + } else { + reportSql = config.reportSql(); } ReportWorker reportWorker = new ReportWorker(config, queue); reportWorker.init(); diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-json/pom.xml b/tika-pipes/tika-pipes-plugins/tika-pipes-json/pom.xml index 9661e09c7d..4e0eb2f2e4 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-json/pom.xml +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-json/pom.xml @@ -31,20 +31,15 @@ <url>https://tika.apache.org/</url> <properties> - <plugin.excluded.artifactIds>tika-core,tika-pipes-api,tika-serialization,tika-plugins-core,tika-pipes-iterator-commons</plugin.excluded.artifactIds> - <plugin.excluded.groupIds>org.apache.logging.log4j,org.slf4j</plugin.excluded.groupIds> </properties> <dependencies> - <dependency> - <groupId>${project.groupId}</groupId> - <artifactId>tika-pipes-iterator-commons</artifactId> - <version>${project.version}</version> - </dependency> + <!-- host framework, parent-loaded --> <dependency> <groupId>${project.groupId}</groupId> <artifactId>tika-pipes-core</artifactId> <version>${project.version}</version> + <scope>provided</scope> </dependency> </dependencies> <build> @@ -71,8 +66,6 @@ <configuration> <outputDirectory>target/lib</outputDirectory> <includeScope>runtime</includeScope> - <excludeArtifactIds>${plugin.excluded.artifactIds}</excludeArtifactIds> - <excludeGroupIds>${plugin.excluded.groupIds}</excludeGroupIds> </configuration> </execution> </executions> diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-kafka/pom.xml b/tika-pipes/tika-pipes-plugins/tika-pipes-kafka/pom.xml index f619bf234e..61d518ab03 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-kafka/pom.xml +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-kafka/pom.xml @@ -31,8 +31,6 @@ <properties> <!-- Never include the core artifacts in your plugin lib directory. If you do, it will cause the classloading to get messed up when finding your plugins. --> - <plugin.excluded.artifactIds>tika-core,tika-pipes-api,tika-serialization,tika-plugins-core</plugin.excluded.artifactIds> - <plugin.excluded.groupIds>org.apache.logging.log4j,org.slf4j</plugin.excluded.groupIds> </properties> <dependencies> @@ -42,11 +40,6 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> - <dependency> - <groupId>${project.groupId}</groupId> - <artifactId>tika-pipes-iterator-commons</artifactId> - <version>${project.version}</version> - </dependency> <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> @@ -84,8 +77,6 @@ <configuration> <outputDirectory>${project.build.directory}/lib</outputDirectory> <includeScope>runtime</includeScope> - <excludeArtifactIds>${plugin.excluded.artifactIds}</excludeArtifactIds> - <excludeGroupIds>${plugin.excluded.groupIds}</excludeGroupIds> </configuration> </execution> </executions> diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-microsoft-graph/pom.xml b/tika-pipes/tika-pipes-plugins/tika-pipes-microsoft-graph/pom.xml index 833dc53541..e9fefe7729 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-microsoft-graph/pom.xml +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-microsoft-graph/pom.xml @@ -35,8 +35,6 @@ <microsoft-kiota.version>1.9.3</microsoft-kiota.version> <!-- Never include the core artifacts in your plugin lib directory. If you do, it will cause the classloading to get messed up when finding your plugins. --> - <plugin.excluded.artifactIds>tika-core,tika-pipes-api,tika-serialization,tika-plugins-core</plugin.excluded.artifactIds> - <plugin.excluded.groupIds>org.apache.logging.log4j,org.slf4j</plugin.excluded.groupIds> </properties> <dependencyManagement> @@ -166,8 +164,6 @@ <configuration> <outputDirectory>target/lib</outputDirectory> <includeScope>runtime</includeScope> - <excludeArtifactIds>${plugin.excluded.artifactIds}</excludeArtifactIds> - <excludeGroupIds>${plugin.excluded.groupIds}</excludeGroupIds> </configuration> </execution> </executions> diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-opensearch/pom.xml b/tika-pipes/tika-pipes-plugins/tika-pipes-opensearch/pom.xml index 4cca9007da..068f6c1eaa 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-opensearch/pom.xml +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-opensearch/pom.xml @@ -30,8 +30,6 @@ <properties> <!-- Never include the core artifacts in your plugin lib directory. If you do, it will cause the classloading to get messed up when finding your plugins. --> - <plugin.excluded.artifactIds>tika-core,tika-pipes-api,tika-serialization,tika-plugins-core</plugin.excluded.artifactIds> - <plugin.excluded.groupIds>org.apache.logging.log4j,org.slf4j</plugin.excluded.groupIds> </properties> <dependencies> @@ -41,11 +39,6 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> - <dependency> - <groupId>${project.groupId}</groupId> - <artifactId>tika-pipes-iterator-commons</artifactId> - <version>${project.version}</version> - </dependency> <dependency> <groupId>${project.groupId}</groupId> <artifactId>tika-pipes-reporter-commons</artifactId> @@ -81,8 +74,6 @@ <configuration> <outputDirectory>${project.build.directory}/lib</outputDirectory> <includeScope>runtime</includeScope> - <excludeArtifactIds>${plugin.excluded.artifactIds}</excludeArtifactIds> - <excludeGroupIds>${plugin.excluded.groupIds}</excludeGroupIds> </configuration> </execution> </executions> diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-s3/pom.xml b/tika-pipes/tika-pipes-plugins/tika-pipes-s3/pom.xml index 4335bce1c1..e469318d24 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-s3/pom.xml +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-s3/pom.xml @@ -31,8 +31,6 @@ <properties> <!-- Never include the core artifacts in your plugin lib directory. If you do, it will cause the classloading to get messed up when finding your plugins. --> - <plugin.excluded.artifactIds>tika-core,tika-pipes-api,tika-serialization,tika-plugins-core</plugin.excluded.artifactIds> - <plugin.excluded.groupIds>org.apache.logging.log4j,org.slf4j</plugin.excluded.groupIds> </properties> <dependencies> @@ -42,11 +40,6 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> - <dependency> - <groupId>${project.groupId}</groupId> - <artifactId>tika-pipes-iterator-commons</artifactId> - <version>${project.version}</version> - </dependency> <dependency> <groupId>${project.groupId}</groupId> <artifactId>tika-serialization</artifactId> @@ -88,8 +81,6 @@ <configuration> <outputDirectory>${project.build.directory}/lib</outputDirectory> <includeScope>runtime</includeScope> - <excludeArtifactIds>${plugin.excluded.artifactIds}</excludeArtifactIds> - <excludeGroupIds>${plugin.excluded.groupIds}</excludeGroupIds> </configuration> </execution> </executions> diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java b/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java index d4a671a927..88748941f6 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3Fetcher.java @@ -252,6 +252,8 @@ public class S3Fetcher extends AbstractTikaExtension implements Fetcher, RangeFe tmp = new TemporaryResources(); Path tmpPath = tmp.createTempFile(FilenameUtils.getSuffixFromPath(fetchKey)); Files.copy(s3Object, tmpPath, StandardCopyOption.REPLACE_EXISTING); + // Files.copy does not close its input; the success path must + s3Object.close(); TikaInputStream tis = TikaInputStream.get(tmpPath, metadata, tmp); LOGGER.debug("took {} ms to fetch metadata and copy to local tmp file", System.currentTimeMillis() - start); diff --git a/tika-pipes/tika-pipes-plugins/tika-pipes-solr/pom.xml b/tika-pipes/tika-pipes-plugins/tika-pipes-solr/pom.xml index f23a76b45c..421b4cbb25 100644 --- a/tika-pipes/tika-pipes-plugins/tika-pipes-solr/pom.xml +++ b/tika-pipes/tika-pipes-plugins/tika-pipes-solr/pom.xml @@ -32,16 +32,9 @@ <properties> <!-- You want to make sure you don't load competing dependencies in your plugin or your classloading is going to get messed up when finding your plugins. --> - <plugin.excluded.artifactIds>tika-core,tika-pipes-api,tika-serialization,tika-plugins-core</plugin.excluded.artifactIds> - <plugin.excluded.groupIds>org.apache.logging.log4j,org.slf4j</plugin.excluded.groupIds> </properties> <dependencies> - <dependency> - <groupId>${project.groupId}</groupId> - <artifactId>tika-pipes-iterator-commons</artifactId> - <version>${project.version}</version> - </dependency> <dependency> <groupId>${project.groupId}</groupId> <artifactId>tika-pipes-api</artifactId> @@ -116,8 +109,6 @@ <configuration> <outputDirectory>${project.build.directory}/lib</outputDirectory> <includeScope>runtime</includeScope> - <excludeArtifactIds>${plugin.excluded.artifactIds}</excludeArtifactIds> - <excludeGroupIds>${plugin.excluded.groupIds}</excludeGroupIds> </configuration> </execution> </executions> diff --git a/tika-serialization/src/main/java/org/apache/tika/config/loader/AbstractSpiComponentLoader.java b/tika-serialization/src/main/java/org/apache/tika/config/loader/AbstractSpiComponentLoader.java index eb0787c711..2338756353 100644 --- a/tika-serialization/src/main/java/org/apache/tika/config/loader/AbstractSpiComponentLoader.java +++ b/tika-serialization/src/main/java/org/apache/tika/config/loader/AbstractSpiComponentLoader.java @@ -300,9 +300,14 @@ public abstract class AbstractSpiComponentLoader<T> implements ComponentLoader<T } } + /** + * An unresolvable exclusion stops the world (TIKA-3268): silently ignoring it + * would leave a deliberately disabled component enabled. + */ @SuppressWarnings("unchecked") private Set<Class<? extends T>> parseExclusions(JsonNode configNode, - LoaderContext context) { + LoaderContext context) + throws TikaConfigException { Set<Class<? extends T>> exclusions = new HashSet<>(); if (configNode == null || !configNode.isObject()) { @@ -327,7 +332,12 @@ public abstract class AbstractSpiComponentLoader<T> implements ComponentLoader<T exclusions.add((Class<? extends T>) clazz); LOG.debug("Excluding {} from SPI: {}", sectionName, typeName); } catch (ClassNotFoundException e) { - LOG.warn("Unknown {} in exclude list: {}", sectionName, typeName); + throw new TikaConfigException( + "Unknown " + sectionName + " in exclude list: '" + typeName + "'. " + + "The exclusion cannot be applied, so the component would stay " + + "enabled. In 4.x, components are named by their registered " + + "component name (e.g. 'pdf-parser'), not by class name; see the " + + "4.x migration guide if this config came from 3.x.", e); } } diff --git a/tika-serialization/src/main/java/org/apache/tika/serialization/ConfigDeserializer.java b/tika-serialization/src/main/java/org/apache/tika/serialization/ConfigDeserializer.java index 2be514b852..ba6db2a0f4 100644 --- a/tika-serialization/src/main/java/org/apache/tika/serialization/ConfigDeserializer.java +++ b/tika-serialization/src/main/java/org/apache/tika/serialization/ConfigDeserializer.java @@ -97,12 +97,11 @@ public class ConfigDeserializer { // Deserialize and merge with default T config = JsonMergeUtils.mergeWithDefaults(MAPPER, jsonConfig.json(), configClass, defaultConfig); - // Cache in resolved configs + // Cache per key only. A class-keyed write would leak this config to every + // component sharing the class (e.g. the three VLM parsers all bind VLMOCRConfig, + // so one provider's base URL and API key would reach the others). context.setResolvedConfig(configKey, config); - // Also set in main context so other components can find it via parseContext.get(configClass) - context.set(configClass, config); - return config; } diff --git a/tika-serialization/src/test/java/org/apache/tika/config/loader/TikaLoaderTest.java b/tika-serialization/src/test/java/org/apache/tika/config/loader/TikaLoaderTest.java index 0dc9c36203..d294ac203a 100644 --- a/tika-serialization/src/test/java/org/apache/tika/config/loader/TikaLoaderTest.java +++ b/tika-serialization/src/test/java/org/apache/tika/config/loader/TikaLoaderTest.java @@ -19,6 +19,7 @@ package org.apache.tika.config.loader; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.net.URL; @@ -30,6 +31,7 @@ import org.junit.jupiter.api.Test; import org.xml.sax.helpers.DefaultHandler; import org.apache.tika.config.EmbeddedLimits; +import org.apache.tika.exception.TikaConfigException; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.HttpHeaders; import org.apache.tika.metadata.Metadata; @@ -252,6 +254,19 @@ public class TikaLoaderTest { "Should NOT support application/test+minimal"); } + /** TIKA-3268: an unresolvable exclusion must fail loudly, not warn. */ + @Test + public void testUnknownExclusionFailsLoudly() throws Exception { + URL configUrl = + getClass().getResource("/configs/test-default-parser-unknown-exclusion.json"); + Path configPath = Path.of(configUrl.toURI()); + + TikaConfigException e = assertThrows(TikaConfigException.class, + () -> TikaLoader.load(configPath).get(Parser.class)); + assertTrue(e.getMessage().contains("no-such-parser"), + "the message must name the unresolvable entry, got: " + e.getMessage()); + } + @Test public void testDefaultParserWithExclusions() throws Exception { // Config has "default-parser" with exclude list diff --git a/tika-serialization/src/test/java/org/apache/tika/serialization/ConfigDeserializerTest.java b/tika-serialization/src/test/java/org/apache/tika/serialization/ConfigDeserializerTest.java index ead8d15da3..7bc3a426da 100644 --- a/tika-serialization/src/test/java/org/apache/tika/serialization/ConfigDeserializerTest.java +++ b/tika-serialization/src/test/java/org/apache/tika/serialization/ConfigDeserializerTest.java @@ -286,4 +286,42 @@ public class ConfigDeserializerTest { assertTrue(ParseContextConfig.hasConfig(context, "parser-a")); assertFalse(ParseContextConfig.hasConfig(context, "parser-b")); } + + /** + * Components with distinct config keys may share one config class (the VLM + * parsers all bind VLMOCRConfig). Resolving one key must never leak that + * config -- base URL, API key -- to another key's component. + */ + @Test + public void testSharedConfigClassDoesNotCrossContaminate() throws Exception { + ParseContext context = new ParseContext(); + context.setJsonConfig("provider-a", "{\"name\":\"a\",\"value\":1}"); + context.setJsonConfig("provider-b", "{\"name\":\"b\",\"value\":2}"); + + TestConfig a = ParseContextConfig.getConfig( + context, "provider-a", TestConfig.class, new TestConfig()); + TestConfig b = ParseContextConfig.getConfig( + context, "provider-b", TestConfig.class, new TestConfig()); + + assertEquals("a", a.getName()); + assertEquals("b", b.getName()); + assertEquals(2, b.getValue()); + + // and the second lookup of each key returns its own cached instance + assertEquals("a", ParseContextConfig.getConfig( + context, "provider-a", TestConfig.class, new TestConfig()).getName()); + } + + /** A programmatic class-keyed override still applies when a key has no JSON config. */ + @Test + public void testProgrammaticClassOverrideStillHonored() throws Exception { + ParseContext context = new ParseContext(); + TestConfig mine = new TestConfig(); + mine.setName("programmatic"); + context.set(TestConfig.class, mine); + + TestConfig got = ParseContextConfig.getConfig( + context, "no-json-for-this-key", TestConfig.class, new TestConfig()); + assertEquals("programmatic", got.getName()); + } } diff --git a/tika-serialization/src/test/resources/configs/test-default-parser-unknown-exclusion.json b/tika-serialization/src/test/resources/configs/test-default-parser-unknown-exclusion.json new file mode 100644 index 0000000000..5b372abcc2 --- /dev/null +++ b/tika-serialization/src/test/resources/configs/test-default-parser-unknown-exclusion.json @@ -0,0 +1,9 @@ +{ + "parsers": [ + { + "default-parser": { + "exclude": ["no-such-parser"] + } + } + ] +} diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java index 361304287b..ceb9c72b42 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java @@ -286,9 +286,7 @@ public class TikaResource { } } - // Create TikaInputStream and spool to temp file immediately. - // This ensures the data is captured before any other processing - // and TikaInputStream handles temp file cleanup automatically. + // Lazy TikaInputStream: nothing is spooled until a consumer needs a file. TikaInputStream tis = TikaInputStream.get(fileAtt.getObject(InputStream.class)); boolean handedOff = false; try { @@ -442,7 +440,7 @@ public class TikaResource { /** Parse document and return HTML content. */ @PUT @Consumes("*/*") - @Produces("text/html") + @Produces("text/html;charset=UTF-8") @Path("html") public Response getHtml(final InputStream is, @Context HttpHeaders httpHeaders) throws IOException { @@ -452,7 +450,7 @@ public class TikaResource { /** Parse document and return XML content. */ @PUT @Consumes("*/*") - @Produces("text/xml") + @Produces("text/xml;charset=UTF-8") @Path("xml") public Response getXml(final InputStream is, @Context HttpHeaders httpHeaders) throws IOException { @@ -543,7 +541,7 @@ public class TikaResource { /** Multipart document with optional config; returns HTML. */ @POST @Consumes("multipart/form-data") - @Produces("text/html") + @Produces("text/html;charset=UTF-8") @Path("config/html") public Response postHtml(List<Attachment> attachments, @Context HttpHeaders httpHeaders) throws IOException, TikaConfigException { @@ -553,7 +551,7 @@ public class TikaResource { /** Multipart document with optional config; returns XML. */ @POST @Consumes("multipart/form-data") - @Produces("text/xml") + @Produces("text/xml;charset=UTF-8") @Path("config/xml") public Response postXml(List<Attachment> attachments, @Context HttpHeaders httpHeaders) throws IOException, TikaConfigException { diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/writer/CSVMessageBodyWriter.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/writer/CSVMessageBodyWriter.java index afe4275de8..d99f43d7d1 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/writer/CSVMessageBodyWriter.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/writer/CSVMessageBodyWriter.java @@ -38,7 +38,7 @@ import org.apache.commons.csv.CSVPrinter; import org.apache.tika.metadata.Metadata; @Provider -@Produces("text/csv") +@Produces("text/csv;charset=UTF-8") public class CSVMessageBodyWriter implements MessageBodyWriter<Metadata> { public boolean isWriteable(Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType) {
