This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4826 in repository https://gitbox.apache.org/repos/asf/tika.git
commit cade330abe377a8d997a6ac25b38db9a196a30a7 Author: tallison <[email protected]> AuthorDate: Tue Sep 1 11:38:18 2026 -0400 TIKA-4826: stop interning untrusted content types into the MimeTypes registry; deprecate forName --- CHANGES.txt | 9 +++++ .../java/org/apache/tika/io/FilenameUtils.java | 10 ++--- .../main/java/org/apache/tika/mime/MimeTypes.java | 43 ++++++++++++++++++++-- .../java/org/apache/tika/mime/MimeTypesReader.java | 2 +- .../mime/ProbabilisticMimeDetectionSelector.java | 2 +- .../java/org/apache/tika/io/FilenameUtilsTest.java | 14 +++++++ .../org/apache/tika/mime/MimeDetectionTest.java | 20 ++++++++++ .../org/apache/tika/eval/app/db/MimeBuffer.java | 6 ++- .../apache/tika/example/ExtractEmbeddedFiles.java | 9 +++-- .../apache/tika/parser/mail/RFC822ParserTest.java | 15 ++++++++ .../testRFC822-unknown-content-type.eml | 21 +++++++++++ .../parser/microsoft/AbstractPOIFSExtractor.java | 9 +++-- 12 files changed, 143 insertions(+), 17 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index ce1baa8f55..9165c4b9f0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,14 @@ Release 4.1.0 - unreleased + * Content-type hints from untrusted input (request headers, document + metadata) are no longer interned into the shared MimeTypes registry + during detection and extension lookup, closing a slow unbounded + memory growth path. MimeTypes.forName is deprecated (unchanged + behavior) in favor of forNameAndRegister, which names the + registration side effect, and getRegisteredMimeType for + side-effect-free lookups; forName will be removed in Tika 5 + (TIKA-4826). + * Add "exception-reporting" parse-context config to redact and bound exception text in metadata, tika-server error bodies and pipes/grpc messages; FileSystemEmitter writes atomically (TIKA-4848). diff --git a/tika-core/src/main/java/org/apache/tika/io/FilenameUtils.java b/tika-core/src/main/java/org/apache/tika/io/FilenameUtils.java index 53ac50e968..27b80340e1 100644 --- a/tika-core/src/main/java/org/apache/tika/io/FilenameUtils.java +++ b/tika-core/src/main/java/org/apache/tika/io/FilenameUtils.java @@ -27,6 +27,7 @@ import org.apache.tika.extractor.EmbeddedDocumentUtil; import org.apache.tika.metadata.HttpHeaders; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.mime.MimeType; import org.apache.tika.mime.MimeTypeException; import org.apache.tika.mime.MimeTypes; import org.apache.tika.utils.StringUtils; @@ -405,11 +406,10 @@ public class FilenameUtils { private static String lookupExtension(String mime) { try { - String ext = MIME_TYPES - .forName(mime) - .getExtension(); - if (!StringUtils.isBlank(ext)) { - return ext; + // not forName: don't intern untrusted types into the registry (TIKA-4826) + MimeType mimeType = MIME_TYPES.getRegisteredMimeType(mime); + if (mimeType != null && !StringUtils.isBlank(mimeType.getExtension())) { + return mimeType.getExtension(); } } catch (MimeTypeException e) { //swallow diff --git a/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java b/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java index c178729cde..83222f9f66 100644 --- a/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java +++ b/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java @@ -206,7 +206,7 @@ public final class MimeTypes implements Detector, Serializable { */ @Deprecated public MimeType getMimeType(File file) throws MimeTypeException, IOException { - return forName(new Tika(this).detect(file)); + return forNameAndRegister(new Tika(this).detect(file)); } /** @@ -296,7 +296,8 @@ public final class MimeTypes implements Detector, Serializable { try { TextDetector detector = new TextDetector(getMinLength()); try (TikaInputStream tis = TikaInputStream.get(data)) { - MimeType type = forName(detector.detect(tis, new Metadata(), new ParseContext()).toString()); + MimeType type = forNameAndRegister( + detector.detect(tis, new Metadata(), new ParseContext()).toString()); return Collections.singletonList(type); } } catch (Exception e) { @@ -347,8 +348,27 @@ public final class MimeTypes implements Detector, Serializable { * @param name media type name (case-insensitive) * @return the registered media type with the given name or alias * @throws MimeTypeException if the given media type name is invalid + * @deprecated since 4.1.0: the name hides the registration side effect. + * Use {@link #forNameAndRegister(String)}; on untrusted input, + * {@link #forNameWithoutRegistration(String)} (drop-in) or + * {@link #getRegisteredMimeType(String)} (null if unknown). Removal in Tika 5. */ + @Deprecated public MimeType forName(String name) throws MimeTypeException { + return forNameAndRegister(name); + } + + /** + * Returns the registered media type with the given name (or alias), + * permanently registering it if valid but unknown. Never call with + * untrusted names — each unknown name grows the registry (TIKA-4826); + * use {@link #forNameWithoutRegistration(String)} instead. + * + * @param name media type name (case-insensitive) + * @return the registered media type with the given name or alias + * @throws MimeTypeException if the given media type name is invalid + */ + public MimeType forNameAndRegister(String name) throws MimeTypeException { MediaType type = MediaType.parse(name); if (type == null) { throw new MimeTypeException("Invalid media type name: " + name); @@ -371,6 +391,23 @@ public final class MimeTypes implements Detector, Serializable { return mime; } + /** + * Like {@link #forNameAndRegister(String)}, but never modifies the + * registry: unknown names get a transient MimeType (TIKA-4826). + * + * @param name media type name (case-insensitive) + * @return the registered media type, or a transient one if unknown + * @throws MimeTypeException if the given media type name is invalid + */ + public MimeType forNameWithoutRegistration(String name) throws MimeTypeException { + MediaType type = MediaType.parse(name); + if (type == null) { + throw new MimeTypeException("Invalid media type name: " + name); + } + MimeType mime = types.get(registry.normalize(type)); + return mime != null ? mime : new MimeType(type); + } + /** * Returns the registered, normalised media type with the given name (or alias). * @@ -574,7 +611,7 @@ public final class MimeTypes implements Detector, Serializable { String typeName = metadata.get(HttpHeaders.CONTENT_TYPE); if (typeName != null) { try { - MimeType hint = forName(typeName); + MimeType hint = forNameWithoutRegistration(typeName); possibleTypes = applyHint(possibleTypes, hint); } catch (MimeTypeException e) { // Malformed type name, ignore diff --git a/tika-core/src/main/java/org/apache/tika/mime/MimeTypesReader.java b/tika-core/src/main/java/org/apache/tika/mime/MimeTypesReader.java index 04c780fa1a..b991e29edc 100644 --- a/tika-core/src/main/java/org/apache/tika/mime/MimeTypesReader.java +++ b/tika-core/src/main/java/org/apache/tika/mime/MimeTypesReader.java @@ -266,7 +266,7 @@ public class MimeTypesReader extends DefaultHandler implements MimeTypesReaderMe String interpretedAttr = attributes.getValue(INTERPRETED_ATTR); boolean interpreted = "true".equals(interpretedAttr); try { - type = types.forName(name); + type = types.forNameAndRegister(name); type.setInterpreted(interpreted); } catch (MimeTypeException e) { handleMimeError(name, e, qName, attributes); diff --git a/tika-core/src/main/java/org/apache/tika/mime/ProbabilisticMimeDetectionSelector.java b/tika-core/src/main/java/org/apache/tika/mime/ProbabilisticMimeDetectionSelector.java index 6bb3cb02cc..d42f4577f1 100644 --- a/tika-core/src/main/java/org/apache/tika/mime/ProbabilisticMimeDetectionSelector.java +++ b/tika-core/src/main/java/org/apache/tika/mime/ProbabilisticMimeDetectionSelector.java @@ -176,7 +176,7 @@ public class ProbabilisticMimeDetectionSelector implements Detector { if (typeName != null) { try { // MimeType hint = forName(typeName); - metaHint = mimeTypes.forName(typeName); + metaHint = mimeTypes.forNameWithoutRegistration(typeName); // possibleTypes = applyHint(possibleTypes, hint); } catch (MimeTypeException e) { // Malformed type name, ignore diff --git a/tika-core/src/test/java/org/apache/tika/io/FilenameUtilsTest.java b/tika-core/src/test/java/org/apache/tika/io/FilenameUtilsTest.java index 014096ea1b..1c4ec7675a 100644 --- a/tika-core/src/test/java/org/apache/tika/io/FilenameUtilsTest.java +++ b/tika-core/src/test/java/org/apache/tika/io/FilenameUtilsTest.java @@ -31,6 +31,7 @@ import org.junit.jupiter.api.Test; import org.apache.tika.metadata.HttpHeaders; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.mime.MimeTypes; import org.apache.tika.utils.StringUtils; public class FilenameUtilsTest { @@ -257,6 +258,19 @@ public class FilenameUtilsTest { return FilenameUtils.getSanitizedEmbeddedFileName(getMetadata(name, mimeType), ".bin", 50); } + @Test + public void testCalculateExtension() throws Exception { + Metadata metadata = new Metadata(); + metadata.set(HttpHeaders.CONTENT_TYPE, "application/pdf"); + assertEquals(".pdf", FilenameUtils.calculateExtension(metadata, ".tmp")); + + // TIKA-4826: unknown types must not grow the shared registry + String unknown = "application/x-tika-4826-filenameutils"; + metadata.set(HttpHeaders.CONTENT_TYPE, unknown); + assertEquals(".bin", FilenameUtils.calculateExtension(metadata, ".tmp")); + assertNull(MimeTypes.getDefaultMimeTypes().getRegisteredMimeType(unknown)); + } + private String sanitizeFilename(String name) { return FilenameUtils.getSanitizedEmbeddedFileName(getMetadata(name), ".bin", 50); } diff --git a/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java b/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java index 5748f185c1..af928a9361 100644 --- a/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java +++ b/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java @@ -21,6 +21,7 @@ import static java.nio.charset.StandardCharsets.UTF_16LE; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; @@ -108,6 +109,25 @@ public class MimeDetectionTest { testUrlWithoutContent("text/x-php", "ftp://test.com/test.php";); } + @Test + public void testUnknownContentTypeHintNotInterned() throws Exception { + // TIKA-4826: unknown content-type hints must not grow the registry + MimeTypes mimeTypes = MimeTypesFactory.create("tika-mimetypes.xml"); + String unknown = "application/x-tika-4826-unknown"; + Metadata metadata = new Metadata(); + metadata.set(HttpHeaders.CONTENT_TYPE, unknown); + assertEquals(unknown, mimeTypes.detect(null, metadata, new ParseContext()).toString()); + assertNull(mimeTypes.getRegisteredMimeType(unknown)); + + ProbabilisticMimeDetectionSelector selector = + new ProbabilisticMimeDetectionSelector(mimeTypes); + String unknown2 = "application/x-tika-4826-unknown2"; + metadata = new Metadata(); + metadata.set(HttpHeaders.CONTENT_TYPE, unknown2); + selector.detect(null, metadata, new ParseContext()); + assertNull(mimeTypes.getRegisteredMimeType(unknown2)); + } + @Test public void testByteOrderMark() throws Exception { try (TikaInputStream tis = TikaInputStream.get("\ufefftest".getBytes(UTF_16LE))) { diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/db/MimeBuffer.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/db/MimeBuffer.java index 9541020541..39d741fd9d 100644 --- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/db/MimeBuffer.java +++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/db/MimeBuffer.java @@ -101,7 +101,11 @@ public class MimeBuffer extends AbstractDBBuffer { * @throws MimeTypeException thrown if MimeTypes can't parse the contentType */ public static String getExtension(String contentType, MimeTypes mimeTypes) throws MimeTypeException { - MimeType mime = mimeTypes.forName(contentType); + // not forName: don't intern untrusted types into the registry (TIKA-4826) + MimeType mime = mimeTypes.getRegisteredMimeType(contentType); + if (mime == null) { + return tryTextyTypes(MediaType.parse(contentType)); + } return getExtension(mime); } diff --git a/tika-example/src/main/java/org/apache/tika/example/ExtractEmbeddedFiles.java b/tika-example/src/main/java/org/apache/tika/example/ExtractEmbeddedFiles.java index 6467483b04..fb9b0745c5 100644 --- a/tika-example/src/main/java/org/apache/tika/example/ExtractEmbeddedFiles.java +++ b/tika-example/src/main/java/org/apache/tika/example/ExtractEmbeddedFiles.java @@ -100,10 +100,13 @@ public class ExtractEmbeddedFiles { if (name.indexOf('.') == -1 && contentType != null) { try { - name += tikaLoader + // not forName: don't intern untrusted types into the registry (TIKA-4826) + MimeType mimeType = tikaLoader .getMimeTypes() - .forName(contentType.toString()) - .getExtension(); + .getRegisteredMimeType(contentType.toString()); + if (mimeType != null) { + name += mimeType.getExtension(); + } } catch (MimeTypeException e) { e.printStackTrace(); } diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java index 17412fed55..8f9e69a7fe 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/test/java/org/apache/tika/parser/mail/RFC822ParserTest.java @@ -20,6 +20,7 @@ import static java.nio.charset.StandardCharsets.US_ASCII; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; @@ -51,6 +52,7 @@ import org.apache.tika.metadata.Message; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.mime.MediaType; +import org.apache.tika.mime.MimeTypes; import org.apache.tika.parser.AutoDetectParser; import org.apache.tika.parser.ParseContext; import org.apache.tika.parser.Parser; @@ -479,6 +481,19 @@ public class RFC822ParserTest extends TikaTest { metadataList.get(1).get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH)); } + @Test + public void testUnknownContentTypesNotInterned() throws Exception { + // TIKA-4826 tripwire: a full parse must not grow the shared registry + MimeTypes mimeTypes = MimeTypes.getDefaultMimeTypes(); + int before = mimeTypes.getMediaTypeRegistry().getTypes().size(); + List<Metadata> metadataList = + getRecursiveMetadata("testRFC822-unknown-content-type.eml"); + assertEquals(3, metadataList.size()); + assertNull(mimeTypes.getRegisteredMimeType("application/x-tika-4826-eml-unknown")); + assertNull(mimeTypes.getRegisteredMimeType("application/x-tika-4826-eml-unknown-noname")); + assertEquals(before, mimeTypes.getMediaTypeRegistry().getTypes().size()); + } + @Test public void testMixedRelatedMultipart() throws Exception { /* diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/test/resources/test-documents/testRFC822-unknown-content-type.eml b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/test/resources/test-documents/testRFC822-unknown-content-type.eml new file mode 100644 index 0000000000..552430ea7d --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-mail-module/src/test/resources/test-documents/testRFC822-unknown-content-type.eml @@ -0,0 +1,21 @@ +From: [email protected] +To: [email protected] +Subject: TIKA-4826 unknown content types +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="tika4826boundary" + +--tika4826boundary +Content-Type: text/plain; charset=us-ascii + +Body text. +--tika4826boundary +Content-Type: application/x-tika-4826-eml-unknown +Content-Disposition: attachment; filename="payload.dat" + +not really binary, but an unknown declared content type +--tika4826boundary +Content-Type: application/x-tika-4826-eml-unknown-noname +Content-Disposition: attachment + +a second attachment with no filename +--tika4826boundary-- diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java index 28dbffa236..88d9652783 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/AbstractPOIFSExtractor.java @@ -314,9 +314,12 @@ abstract class AbstractPOIFSExtractor { MediaType mediaType = getDetector().detect(tis, metadata, context); String extension = type.getExtension(); try { - MimeType mimeType = - EmbeddedDocumentUtil.getMimeTypes(context).forName(mediaType.toString()); - extension = mimeType.getExtension(); + // not forName: don't intern untrusted types into the registry (TIKA-4826) + MimeType mimeType = EmbeddedDocumentUtil.getMimeTypes(context) + .getRegisteredMimeType(mediaType.toString()); + if (mimeType != null) { + extension = mimeType.getExtension(); + } } catch (MimeTypeException mte) { // No details on this type are known }
