This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch update-serialization-docs in repository https://gitbox.apache.org/repos/asf/tika.git
commit 136593b76d28fbb12a80b42c558cf1ee66303878 Merge: 193d69cff0 39f035cc77 Author: tallison <[email protected]> AuthorDate: Wed Jul 29 21:31:19 2026 -0400 Merge origin/main into update-serialization-docs .github/scripts/check_javadoc_sourcepath.py | 97 +++ .github/scripts/check_split_packages.py | 69 +++ .github/workflows/docker-release.yml | 18 + .github/workflows/docker-snapshot.yml | 18 + .github/workflows/split-packages.yml | 33 ++ .gitignore | 4 + .skills/dev.md | 11 + .skills/metadata-schema.md | 77 +++ .skills/tika-eval-encoding-regression.md | 8 +- .skills/tika-eval-h2-query.md | 2 +- .skills/update-site-for-release/SKILL.md | 346 +++++++++++ .skills/update-site-for-release/scripts/README.md | 54 ++ .../scripts/extract-tika-contribs.py | 252 ++++++++ .../scripts/extract-tika-issues.py | 152 +++++ .../scripts/scaffold-stable-version.sh | 65 ++ CHANGES.txt | 17 + docs/modules/ROOT/examples/detector-pkcs7.json | 6 + .../ROOT/pages/advanced/embedded-documents.adoc | 36 +- .../advanced/integration-testing/tika-app.adoc | 4 +- .../integration-testing/tika-eval-regression.adoc | 6 +- .../advanced/integration-testing/tika-server.adoc | 2 +- .../ROOT/pages/advanced/language-detection.adoc | 2 +- .../ROOT/pages/advanced/setting-limits.adoc | 18 +- .../modules/ROOT/pages/advanced/zip-detection.adoc | 4 +- .../ROOT/pages/configuration/detectors.adoc | 57 ++ .../ROOT/pages/configuration/digesters.adoc | 4 +- docs/modules/ROOT/pages/configuration/index.adoc | 1 + .../ROOT/pages/developers/serialization.adoc | 2 +- .../pages/maintainers/release-guides/tika.adoc | 3 + docs/modules/ROOT/pages/maintainers/site.adoc | 54 ++ .../pages/migration-to-4x/chunk-strategies.adoc | 24 +- .../pages/migration-to-4x/metadata-changes-4x.adoc | 232 +++++++- docs/modules/ROOT/pages/pipes/parse-modes.adoc | 18 +- docs/modules/ROOT/pages/using-tika/grpc/index.adoc | 9 +- .../ROOT/pages/using-tika/server/index.adoc | 2 +- pom.xml | 1 + .../tika/{config => annotation}/TikaComponent.java | 2 +- .../tika/annotation/TikaComponentProcessor.java | 3 +- .../java/org/apache/tika/cli/TikaCLIAsyncTest.java | 4 +- .../test/java/org/apache/tika/cli/TikaCLITest.java | 20 +- tika-bom/pom.xml | 5 + tika-bundles/tika-bundle-standard/pom.xml | 2 - .../org/apache/tika/config/EmbeddedLimits.java | 2 +- .../java/org/apache/tika/config/OutputLimits.java | 2 +- .../java/org/apache/tika/config/TimeoutLimits.java | 2 +- .../java/org/apache/tika/detect/BOMDetector.java | 2 +- .../org/apache/tika/detect/DefaultDetector.java | 2 +- .../java/org/apache/tika/detect/EmptyDetector.java | 2 +- .../apache/tika/detect/FileCommandDetector.java | 2 +- .../java/org/apache/tika/detect/MagicDetector.java | 37 +- .../tika/detect/MetadataCharsetDetector.java | 2 +- .../tika/detect/OverrideEncodingDetector.java | 2 +- .../org/apache/tika/detect/XmlRootExtractor.java | 6 +- .../java/org/apache/tika/digest/DigestDef.java | 10 +- .../apache/tika/digest/InputStreamDigester.java | 14 +- .../extractor/SkipEmbeddedDocumentSelector.java | 2 +- .../tika/extractor/StandardExtractorFactory.java | 2 +- .../java/org/apache/tika/io/SpoolingStrategy.java | 2 +- .../tika/language/translate/DefaultTranslator.java | 2 +- .../tika/language/translate/EmptyTranslator.java | 2 +- .../apache/tika/metadata/AccessPermissions.java | 18 +- .../main/java/org/apache/tika/metadata/Audio.java | 74 +++ .../main/java/org/apache/tika/metadata/DWG.java | 11 +- .../java/org/apache/tika/metadata/Database.java | 8 +- .../main/java/org/apache/tika/metadata/Google.java | 52 ++ .../main/java/org/apache/tika/metadata/HTML.java | 3 + .../java/org/apache/tika/metadata/HttpHeaders.java | 16 +- .../main/java/org/apache/tika/metadata/MAPI.java | 1 + .../java/org/apache/tika/metadata/Message.java | 46 +- .../java/org/apache/tika/metadata/Metadata.java | 88 ++- .../main/java/org/apache/tika/metadata/Office.java | 9 +- .../tika/metadata/OfficeOpenXMLExtended.java | 4 +- .../main/java/org/apache/tika/metadata/PDF.java | 72 +-- .../main/java/org/apache/tika/metadata/PST.java | 4 +- .../apache/tika/metadata/PassthroughPrefix.java | 79 +++ .../java/org/apache/tika/metadata/Photoshop.java | 3 + .../java/org/apache/tika/metadata/QuattroPro.java | 10 +- .../java/org/apache/tika/metadata/QuickTime.java | 38 ++ .../java/org/apache/tika/metadata/RTFMetadata.java | 29 +- .../java/org/apache/tika/metadata/Rendering.java | 4 +- .../apache/tika/metadata/ReservedNamespaces.java | 45 ++ .../main/java/org/apache/tika/metadata/TIFF.java | 12 + .../apache/tika/metadata/TikaCoreProperties.java | 125 ++-- .../org/apache/tika/metadata/TikaPagedText.java | 8 +- .../java/org/apache/tika/metadata/WordPerfect.java | 14 +- .../main/java/org/apache/tika/metadata/XMPMM.java | 14 + .../java/org/apache/tika/metadata/XMPTIFF.java | 70 +++ .../main/java/org/apache/tika/metadata/Zip.java | 22 +- .../filter/CaptureGroupMetadataFilter.java | 2 +- .../ClearByAttachmentTypeMetadataFilter.java | 2 +- .../metadata/filter/CompositeMetadataFilter.java | 2 +- .../filter/DateNormalizingMetadataFilter.java | 2 +- .../filter/ExcludeFieldMetadataFilter.java | 2 +- .../metadata/filter/FieldNameMappingFilter.java | 2 +- .../metadata/filter/GeoPointMetadataFilter.java | 2 +- .../filter/IncludeFieldMetadataFilter.java | 2 +- .../tika/metadata/filter/MetadataFilter.java | 26 +- .../tika/metadata/filter/MetadataFilterBase.java | 3 +- .../apache/tika/metadata/filter/NoOpFilter.java | 4 +- .../filter/RemoveByMimeMetadataFilter.java | 4 +- .../writefilter/StandardMetadataLimiter.java | 8 +- .../StandardMetadataLimiterFactory.java | 2 +- .../main/java/org/apache/tika/mime/MimeTypes.java | 2 +- .../java/org/apache/tika/mime/MimeTypesReader.java | 2 +- .../apache/tika/parser/AutoDetectParserConfig.java | 2 +- .../org/apache/tika/parser/CompositeParser.java | 2 +- .../java/org/apache/tika/parser/DefaultParser.java | 2 +- .../java/org/apache/tika/parser/EmptyParser.java | 2 +- .../java/org/apache/tika/parser/ErrorParser.java | 2 +- .../org/apache/tika/parser/RegexCaptureParser.java | 2 +- .../apache/tika/parser/SimplePasswordProvider.java | 2 +- .../tika/parser/external/ExternalParser.java | 2 +- .../parser/multiple/AbstractMultipleParser.java | 10 +- .../sax/AbstractRecursiveParserWrapperHandler.java | 9 +- .../tika/sax/BasicContentHandlerFactory.java | 2 +- .../java/org/apache/tika/sax/SAXOutputConfig.java | 2 +- .../java/org/apache/tika/utils/ParserUtils.java | 9 +- .../org/apache/tika/mime/tika-mimetypes.xml | 59 +- .../org/apache/tika/detect/MagicDetectorTest.java | 102 ++++ .../tika/digest/InputStreamDigesterTest.java | 46 ++ .../metadata/MetadataInternalKeyGuardTest.java | 134 +++++ .../apache/tika/detect/magika/MagikaDetector.java | 2 +- .../tika/detect/siegfried/SiegfriedDetector.java | 2 +- tika-e2e-tests/pom.xml | 17 - tika-e2e-tests/tika-grpc/pom.xml | 7 - .../org/apache/tika/pipes/ExternalTestBase.java | 43 +- .../pipes/filesystem/FileSystemFetcherTest.java | 24 +- .../tika/pipes/filesystem/HandlerTypeTest.java | 45 +- .../tika/pipes/ignite/IgniteConfigStoreTest.java | 77 +-- .../html/HtmlEncodingDetector.java | 4 +- .../html/TikaHtmlCharsetAliases.java | 2 +- .../StandardHtmlEncodingDetector.java | 4 +- .../org.apache.tika.detect.EncodingDetector | 2 +- .../html/StandardCharsets_unsupported_by_IANA.txt | 0 .../tika/parser/html/HtmlEncodingDetectorTest.java | 3 +- .../html/StandardHtmlEncodingDetectorTest.java | 52 ++ .../tika-encoding-detector-icu4j/pom.xml | 2 +- .../txt => detect/icu4j}/CharsetDetector.java | 2 +- .../{parser/txt => detect/icu4j}/CharsetMatch.java | 2 +- .../txt => detect/icu4j}/CharsetRecog_2022.java | 2 +- .../txt => detect/icu4j}/CharsetRecog_UTF8.java | 2 +- .../txt => detect/icu4j}/CharsetRecog_Unicode.java | 2 +- .../txt => detect/icu4j}/CharsetRecog_mbcs.java | 2 +- .../txt => detect/icu4j}/CharsetRecog_sbcs.java | 2 +- .../txt => detect/icu4j}/CharsetRecognizer.java | 2 +- .../icu4j}/Icu4jEncodingDetector.java | 4 +- .../tika/parser/txt/CharsetDetectorTest.java | 3 +- .../ml/chardetect/MojibusterEncodingDetector.java | 2 +- .../ml/chardetect/StructuralEncodingRules.java | 37 ++ .../Utf16SpecialistEncodingDetector.java | 2 +- .../universal}/UniversalEncodingDetector.java | 4 +- .../universal}/UniversalEncodingListener.java | 2 +- .../org/apache/tika/eval/app/ExtractComparer.java | 4 +- .../org/apache/tika/eval/app/ExtractProfiler.java | 2 +- .../org/apache/tika/eval/app/ProfilerBase.java | 4 +- .../java/org/apache/tika/eval/app/db/Cols.java | 2 +- .../src/main/resources/comparison-reports-pg.xml | 2 +- .../src/main/resources/comparison-reports.xml | 2 +- .../src/test/resources/extracts/TIKA-4446-a.json | 2 +- .../src/test/resources/extracts/TIKA-4446-b.json | 2 +- .../resources/test-dirs/extractsA/file1.pdf.json | 2 +- .../test-dirs/extractsA/file12_es.txt.json | 2 +- .../extractsA/file13_attachANotB.doc.json | 6 +- .../extractsA/file14_diffAttachOrder.json | 20 +- .../resources/test-dirs/extractsA/file15_tags.json | 26 +- .../test-dirs/extractsA/file16_badTags.json | 26 +- .../test-dirs/extractsA/file17_tagsOutOfOrder.json | 26 +- .../test-dirs/extractsA/file2_attachANotB.doc.json | 8 +- .../test-dirs/extractsA/file3_attachBNotA.doc.json | 2 +- .../test-dirs/extractsA/file4_emptyB.pdf.json | 2 +- .../test-dirs/extractsA/file6_accessEx.pdf.json | 18 +- .../test-dirs/extractsA/file7_badJson.pdf.json | 2 +- .../test-dirs/extractsA/file8_IOEx.pdf.json | 18 +- .../resources/test-dirs/extractsB/file1.pdf.json | 2 +- .../test-dirs/extractsB/file12_es.txt.json | 2 +- .../extractsB/file14_diffAttachOrder.json | 20 +- .../test-dirs/extractsB/file2_attachANotB.doc.json | 2 +- .../test-dirs/extractsB/file3_attachBNotA.doc.json | 8 +- .../test-dirs/extractsB/file5_emptyA.pdf.json | 2 +- .../test-dirs/extractsB/file6_accessEx.pdf.json | 18 +- .../test-dirs/extractsB/file8_IOEx.pdf.json | 18 +- .../resources/test-dirs/raw_input/file12_es.txt | 2 +- .../test-dirs/raw_input/file13_attachANotB.doc | 6 +- .../test-dirs/raw_input/file14_diffAttachOrder | 20 +- .../test/resources/test-dirs/raw_input/file15_tags | 26 +- .../resources/test-dirs/raw_input/file16_badTags | 26 +- .../test-dirs/raw_input/file17_tagsOutOfOrder | 26 +- .../eval/core/metadata/TikaEvalMetadataFilter.java | 2 +- .../tika/eval/core/tokens/TikaEvalTokenizer.java | 4 +- .../core/tokens/tools/CommonTokenGenerator.java | 4 +- .../apache/tika/example/TestParsingExample.java | 4 +- tika-grpc/docker-build/Dockerfile | 14 +- .../org/apache/tika/pipes/grpc/TikaGrpcServer.java | 20 +- ...PipesBiDirectionalStreamingIntegrationTest.java | 25 + .../apache/tika/pipes/grpc/TikaGrpcServerTest.java | 2 +- .../tika/pipes/grpc/TikaGrpcServerTlsTest.java | 111 ++++ .../elasticsearch/tests/ElasticsearchTest.java | 21 +- .../resources/elasticsearch/plugins-template.json | 4 +- .../tika/pipes/kafka/tests/TikaPipesKafkaTest.java | 2 +- .../src/test/resources/kafka/plugins-template.json | 4 +- .../pipes/opensearch/tests/OpenSearchTest.java | 22 +- .../resources/opensearch/plugins-template.json | 4 +- .../opensearch/tika-config-opensearch.json | 4 +- .../tika/pipes/s3/tests/S3PipeIntegrationTest.java | 2 +- .../src/test/resources/s3/tika-config-s3.json | 4 +- .../tika-pipes-solr-integration-tests/pom.xml | 2 - ...ipesSolr8Test.java => TikaPipesSolr10Test.java} | 4 +- ...Solr8ZkTest.java => TikaPipesSolr10ZkTest.java} | 5 +- .../pipes/solr/tests/TikaPipesSolrTestBase.java | 5 +- .../src/test/resources/solr/plugins-template.json | 4 +- .../src/test/resources/tika-config-solr-urls.json | 4 +- .../{ => core}/CharSoupFeatureExtractor.java | 4 +- .../charsoup/{ => core}/CharSoupModel.java | 2 +- .../charsoup/{ => core}/FeatureExtractor.java | 2 +- .../{ => core}/SaltedNgramFeatureExtractor.java | 2 +- .../{ => core}/ScriptAwareFeatureExtractor.java | 4 +- .../charsoup/{ => core}/ScriptCategory.java | 2 +- .../{ => core}/ShortTextFeatureExtractor.java | 4 +- .../charsoup/{ => core}/WordTokenizer.java | 2 +- .../charsoup/CharSoupLanguageDetector.java | 7 +- .../charsoup/CharSoupMetadataFilter.java | 4 +- .../charsoup/CharSoupModelRoutingTest.java | 1 - .../langdetect/charsoup/LangIdRegressionTest.java | 1 - .../langdetect/charsoup/SjisLangSignalTest.java | 1 - .../{ => core}/CharSoupFeatureExtractorTest.java | 4 +- .../charsoup/{ => core}/CharSoupModelTest.java | 4 +- .../ScriptAwareFeatureExtractorTest.java | 4 +- .../charsoup/{ => core}/WordTokenizerTest.java | 4 +- .../langdetect/charsoup/tools/AblationRunner.java | 4 +- .../charsoup/tools/BucketSaturationAnalyzer.java | 2 +- .../charsoup/tools/CalibrateConfidence.java | 4 +- .../charsoup/tools/CompareDetectors.java | 2 +- .../langdetect/charsoup/tools/ConfusionDumper.java | 4 +- .../langdetect/charsoup/tools/CrossDomainEval.java | 6 +- .../charsoup/tools/DiagnoseUnknownScript.java | 4 +- .../charsoup/tools/KoreanFalsePositives.java | 6 +- .../langdetect/charsoup/tools/ModelQuantizer.java | 2 +- .../langdetect/charsoup/tools/Phase2SmokeTest.java | 2 +- .../langdetect/charsoup/tools/Phase2Trainer.java | 6 +- .../langdetect/charsoup/tools/PrepareCorpus.java | 2 +- .../langdetect/charsoup/tools/PrintDiscLabels.java | 2 +- .../langdetect/charsoup/tools/QuickF1Eval.java | 4 +- .../charsoup/tools/ResearchFeatureExtractor.java | 12 +- .../charsoup/tools/TrainLanguageModel.java | 6 +- .../tika/langdetect/opennlp/OpenNLPDetector.java | 2 +- .../metadatafilter/OpenNLPMetadataFilter.java | 2 +- .../optimaize/OptimaizeLangDetector.java | 2 +- .../metadatafilter/OptimaizeMetadataFilter.java | 2 +- tika-metadata-schema/README.md | 72 +++ tika-metadata-schema/pom.xml | 95 +++ .../tika/metadata/schema/MetadataKeyValidator.java | 143 +++++ .../tika/metadata/schema/SchemaGenerator.java | 249 ++++++++ .../apache/tika/metadata/metadata-key-fields.json | 654 +++++++++++++++++++++ .../org/apache/tika/metadata/metadata-keys.json | 652 ++++++++++++++++++++ .../tika/metadata/metadata-open-namespaces.json | 23 + .../tika/metadata/schema/MetadataCoverageTest.java | 150 +++++ .../metadata/schema/MetadataFieldTableTest.java | 46 ++ .../metadata/schema/MetadataKeyValidatorTest.java | 90 +++ .../metadata/schema/MetadataNoUnderscoreTest.java | 79 +++ .../tika/metadata/schema/MetadataSchemaTest.java | 59 ++ .../tools/BenchmarkCharsetDetectors.java | 4 +- .../ml/chardetect/tools/EvalCharsetDetectors.java | 4 +- .../tools/BuildJunkAugmentationData.java | 4 +- .../ml/junkdetect/JunkFilterEncodingDetector.java | 48 +- .../junkdetect/JunkFilterEncodingDetectorTest.java | 90 +++ tika-parent/pom.xml | 74 +-- .../tika/parser/ocrencode/EncodeOCRParser.java | 2 +- .../apache/tika/parser/envi/EnviHeaderParser.java | 13 +- .../org/apache/tika/parser/gdal/GDALParser.java | 2 +- .../geoinfo/GeographicInformationParser.java | 9 +- .../org/apache/tika/parser/grib/GribParser.java | 32 +- .../java/org/apache/tika/parser/hdf/HDFParser.java | 4 +- .../apache/tika/parser/isatab/ISArchiveParser.java | 2 +- .../apache/tika/parser/netcdf/NetCDFParser.java | 35 +- .../org/apache/tika/parser/hdf/HDFParserTest.java | 2 +- .../tika/parser/netcdf/NetCDFParserTest.java | 2 +- .../apache/tika/parser/geopkg/GeoPkgParser.java | 2 +- .../apache/tika/parser/sqlite3/SQLite3Parser.java | 2 +- .../tika/inference/AbstractEmbeddingFilter.java | 2 +- .../org/apache/tika/inference/ChunkSerializer.java | 11 +- .../tika/inference/ImageEmbeddingConfig.java | 2 +- .../org/apache/tika/inference/InferenceConfig.java | 2 +- .../apache/tika/inference/JinaEmbeddingFilter.java | 2 +- .../tika/inference/OpenAIEmbeddingFilter.java | 2 +- .../tika/inference/OpenAIImageEmbeddingParser.java | 2 +- .../tika/inference/JinaEmbeddingFilterTest.java | 10 +- .../tika/inference/OpenAIEmbeddingFilterTest.java | 26 +- .../inference/OpenAIImageEmbeddingParserTest.java | 6 +- .../tika-parsers-ml/tika-parser-nlp-module/pom.xml | 2 +- .../tika/parser/ctakes/CTAKESContentHandler.java | 7 +- .../apache/tika/parser/geo/topic/GeoParser.java | 7 +- .../tika/parser/journal/GrobidRESTParser.java | 12 +- .../apache/tika/parser/journal/JournalParser.java | 2 +- .../apache/tika/parser/ner/NamedEntityParser.java | 7 +- .../tika-parser-tess4j-module/pom.xml | 2 +- .../tika/parser/ocr/tess4j/Tess4JParser.java | 2 +- .../tika-parsers-ml/tika-transcribe-aws/pom.xml | 1 - .../parser/transcribe/aws/AmazonTranscribe.java | 2 +- .../apache/tika/parser/vlm/AbstractVLMParser.java | 4 +- .../apache/tika/parser/vlm/ClaudeVLMParser.java | 2 +- .../apache/tika/parser/vlm/GeminiVLMParser.java | 2 +- .../apache/tika/parser/vlm/OpenAIVLMParser.java | 2 +- .../org/apache/tika/config/ConfigExamplesTest.java | 30 + .../tika/config/TikaEncodingDetectorTest.java | 8 +- .../apache/tika/detect/TestDetectorLoading.java | 8 +- .../java/org/apache/tika/mime/TestMimeTypes.java | 7 +- .../tika/parser/AutoDetectParserConfigTest.java | 24 +- .../tika/parser/RecursiveParserWrapperTest.java | 2 +- .../tika/parser/crypto/Pkcs7EndToEndTest.java | 65 ++ .../tika/parser/digest/DigestConfigTest.java | 28 +- .../apache/tika/parser/mail/RFC822ParserTest.java | 3 +- .../parser/microsoft/ooxml/OOXMLParserTest.java | 2 +- .../microsoft/ooxml/OleObjectRegressionTest.java | 6 +- .../parser/microsoft/ooxml/TruncatedOOXMLTest.java | 4 +- .../org/apache/tika/parser/pdf/PDFParserTest.java | 2 +- .../org/apache/tika/parser/pkg/ZipParserTest.java | 6 +- .../DoublingContentHandlerDecoratorFactory.java | 2 +- .../UpcasingContentHandlerDecoratorFactory.java | 2 +- .../resources/config-examples/detector-pkcs7.json | 6 + .../test/resources/configs/TIKA-3137-include.json | 2 +- .../configs/tika-config-write-filter.json | 2 +- .../src/test/resources/test-documents/test.xml.p7m | Bin 0 -> 2504 bytes .../testPKCS7_compressed_def_long.p7z | Bin 0 -> 331 bytes .../test-documents/testPKCS7_signed_data_def.p7m | Bin 0 -> 1574 bytes .../tika-parsers-standard-modules/pom.xml | 1 + .../apache/tika/detect/apple/BPListDetector.java | 2 +- .../tika/parser/apple/AppleSingleFileParser.java | 2 +- .../org/apache/tika/parser/apple/PListParser.java | 2 +- .../tika/parser/iwork/IWorkPackageParser.java | 4 +- .../tika/parser/iwork/NumbersContentHandler.java | 36 +- .../tika/parser/iwork/PagesContentHandler.java | 2 +- .../parser/iwork/iwana/IWork13PackageParser.java | 7 +- .../parser/iwork/iwana/IWork18PackageParser.java | 4 +- .../apache/tika/parser/iwork/IWorkParserTest.java | 6 +- .../tika/detect/{ => mkv}/MatroskaDetector.java | 6 +- .../org/apache/tika/detect/ogg/OggDetector.java | 2 +- .../org/apache/tika/parser/audio/AudioParser.java | 3 +- .../org/apache/tika/parser/audio/MidiParser.java | 2 +- .../apache/tika/parser/audio/NumberAndTotal.java | 67 +++ .../tika/parser/mp3/CompositeTagHandler.java | 9 + .../java/org/apache/tika/parser/mp3/ID3Tags.java | 5 + .../org/apache/tika/parser/mp3/ID3v1Handler.java | 8 + .../org/apache/tika/parser/mp3/ID3v22Handler.java | 16 +- .../org/apache/tika/parser/mp3/ID3v23Handler.java | 16 +- .../org/apache/tika/parser/mp3/ID3v24Handler.java | 16 +- .../org/apache/tika/parser/mp3/ID3v2Frame.java | 74 ++- .../java/org/apache/tika/parser/mp3/Mp3Parser.java | 80 ++- .../org/apache/tika/parser/mp3/MpegStream.java | 67 ++- .../java/org/apache/tika/parser/mp4/MP4Parser.java | 2 +- .../apache/tika/parser/mp4/TikaMp4BoxHandler.java | 231 +++++++- .../apache/tika/parser/mp4/TikaMp4MetaHandler.java | 174 ++++++ .../tika/parser/mp4/TikaMp4SoundHandler.java | 193 ++++++ .../org/apache/tika/parser/mp4/boxes/ISO6709.java | 105 ++++ .../tika/parser/mp4/boxes/TikaUserDataBox.java | 40 +- .../org/apache/tika/parser/ogg/FlacParser.java | 2 +- .../org/apache/tika/parser/ogg/OggAudioParser.java | 82 ++- .../java/org/apache/tika/parser/ogg/OggParser.java | 2 +- .../org/apache/tika/parser/ogg/OpusParser.java | 2 +- .../org/apache/tika/parser/ogg/SpeexParser.java | 2 +- .../org/apache/tika/parser/ogg/TheoraParser.java | 2 +- .../org/apache/tika/parser/ogg/VorbisParser.java | 20 +- .../org/apache/tika/parser/video/FLVParser.java | 11 +- .../apache/tika/detect/MatroskaDetectorTest.java | 3 +- .../apache/tika/parser/audio/AudioParserTest.java | 7 +- .../tika/parser/audio/NumberAndTotalTest.java | 67 +++ .../org/apache/tika/parser/mp3/ID3v2FrameTest.java | 171 ++++++ .../org/apache/tika/parser/mp3/Mp3ParserTest.java | 139 ++++- .../org/apache/tika/parser/mp3/MpegStreamTest.java | 68 +++ .../org/apache/tika/parser/mp4/MP4ParserTest.java | 132 ++++- .../apache/tika/parser/mp4/boxes/ISO6709Test.java | 91 +++ .../parser/ogg/OggAudioParserDurationTest.java | 128 ++++ .../apache/tika/parser/ogg/OggAudioParserTest.java | 117 ++++ .../apache/tika/parser/ogg/VorbisParserTest.java | 104 ++++ .../apache/tika/parser/video/FLVParserTest.java | 10 +- .../resources/test-documents/testMP3cbrInfoTag.mp3 | Bin 0 -> 6681 bytes .../test/resources/test-documents/testMP3id3v2.mp3 | Bin 39577 -> 39602 bytes .../resources/test-documents/testMP3id3v24.mp3 | Bin 39471 -> 39511 bytes .../test/resources/test-documents/testMP3mpeg2.mp3 | Bin 0 -> 8255 bytes .../test/resources/test-documents/testMP3vbr.mp3 | Bin 0 -> 3129 bytes .../test-documents/testMP4_QuickTimeMetadata.mov | Bin 0 -> 2770 bytes .../test-documents/testMP4_StillImageTimeZero.mov | Bin 0 -> 1157 bytes .../test/resources/test-documents/testMP4_drm.m4a | Bin 0 -> 464 bytes .../resources/test-documents/testMP4_esdsFlags.m4a | Bin 0 -> 512 bytes .../test-documents/testMP4_udtaLocation.mp4 | Bin 0 -> 200 bytes .../test/resources/test-documents/testVORBIS.ogg | Bin 0 -> 4241 bytes .../org/apache/tika/parser/dgn/DGN8Parser.java | 2 +- .../java/org/apache/tika/parser/dwg/DWGParser.java | 11 +- .../org/apache/tika/parser/dwg/DWGReadParser.java | 6 +- .../java/org/apache/tika/parser/prt/PRTParser.java | 2 +- .../org/apache/tika/parser/asm/ClassParser.java | 2 +- .../apache/tika/parser/code/SourceCodeParser.java | 2 +- .../tika/parser/executable/ExecutableParser.java | 2 +- .../executable/UniversalExecutableParser.java | 2 +- .../java/org/apache/tika/parser/mat/MatParser.java | 5 +- .../org/apache/tika/parser/sas/SAS7BDATParser.java | 2 +- .../org/apache/tika/parser/mat/MatParserTest.java | 3 +- .../apache/tika/detect/crypto/Pkcs7Detector.java | 27 +- .../apache/tika/parser/crypto/CmsClassifier.java | 175 ++++++ .../org/apache/tika/parser/crypto/Pkcs7Parser.java | 103 +++- .../org/apache/tika/parser/crypto/TSDParser.java | 4 +- .../tika/detect/crypto/Pkcs7DetectorTest.java | 59 ++ .../tika/parser/crypto/CmsClassifierTest.java | 62 ++ .../tika/parser/crypto/Pkcs7DetectionTest.java | 56 ++ .../apache/tika/parser/crypto/Pkcs7ParserTest.java | 61 +- .../src/test/resources/test-documents/test.xml.p7m | Bin 0 -> 2504 bytes .../test-documents/testPKCS7_certs_only_def.p7c | Bin 0 -> 2972 bytes .../test-documents/testPKCS7_certs_only_ind.p7c | Bin 0 -> 2976 bytes .../testPKCS7_compressed_def_long.p7z | Bin 0 -> 331 bytes .../testPKCS7_compressed_def_short.p7z | Bin 0 -> 125 bytes .../test-documents/testPKCS7_compressed_ind.p7z | Bin 0 -> 141 bytes .../resources/test-documents/testPKCS7_digested.p7 | Bin 0 -> 97 bytes .../test-documents/testPKCS7_enveloped_def.p7m | Bin 0 -> 350 bytes .../test-documents/testPKCS7_enveloped_ind.p7m | Bin 0 -> 356 bytes .../test-documents/testPKCS7_signature_def.p7s | Bin 0 -> 1566 bytes .../test-documents/testPKCS7_signature_ind.p7s | Bin 0 -> 1568 bytes .../test-documents/testPKCS7_signed_data_def.p7m | Bin 0 -> 1574 bytes .../test-documents/testPKCS7_signed_data_def.pem | 35 ++ .../test-documents/testPKCS7_signed_data_ind.p7m | Bin 0 -> 1582 bytes .../resources/test-documents/testRSAKEYandCERT.p12 | Bin 0 -> 1717 bytes .../pom.xml | 40 +- .../apache/tika/parser/datauri}/DataURIScheme.java | 12 +- .../datauri}/DataURISchemeParseException.java | 2 +- .../tika/parser/datauri}/DataURISchemeUtil.java | 5 +- .../parser/datauri}/DataURISchemeParserTest.java | 2 +- .../digestutils/BouncyCastleDigesterFactory.java | 2 +- .../parser/digestutils/CommonsDigesterFactory.java | 2 +- .../tika/parser/font/AdobeFontMetricParser.java | 2 +- .../apache/tika/parser/font/TrueTypeParser.java | 4 +- .../apache/tika/parser/font/FontParsersTest.java | 2 +- .../tika-parser-html-module/pom.xml | 5 +- .../org/apache/tika/parser/html/HtmlHandler.java | 5 +- .../org/apache/tika/parser/html/JSoupParser.java | 2 +- .../org/apache/tika/parser/image/BPGParser.java | 9 +- .../org/apache/tika/parser/image/ExtendedXmp.java | 125 ++++ .../org/apache/tika/parser/image/HeifParser.java | 24 +- .../java/org/apache/tika/parser/image/HeifXmp.java | 196 ++++++ .../org/apache/tika/parser/image/ICNSParser.java | 2 +- .../tika/parser/image/ImageMetadataExtractor.java | 97 +-- .../org/apache/tika/parser/image/ImageParser.java | 2 +- .../org/apache/tika/parser/image/ImageXmp.java | 151 +++++ .../org/apache/tika/parser/image/JXLParser.java | 6 +- .../org/apache/tika/parser/image/JpegParser.java | 6 +- .../org/apache/tika/parser/image/PSDParser.java | 8 +- .../org/apache/tika/parser/image/TiffParser.java | 7 +- .../org/apache/tika/parser/image/WebPParser.java | 4 +- .../apache/tika/parser/image/ExtendedXmpTest.java | 76 +++ .../org/apache/tika/parser/image/HeicXmpTest.java | 183 ++++++ .../apache/tika/parser/image/HeifParserTest.java | 1 + .../parser/image/ImageMetadataExtractorTest.java | 24 + .../apache/tika/parser/image/JpegParserTest.java | 5 +- .../tika/parser/image/MotionPhotoXmpTest.java | 74 +++ .../org/apache/tika/parser/image/WebPXmpTest.java | 122 ++++ .../test-documents/testJPEG_MicroVideo.jpg | Bin 0 -> 1082 bytes .../test-documents/testJPEG_MotionPhoto.jpg | Bin 0 -> 1459 bytes .../tika/parser/mail/MailContentHandler.java | 6 +- .../org/apache/tika/parser/mail/RFC822Parser.java | 2 +- .../org/apache/tika/parser/mbox/MboxParser.java | 12 +- .../apache/tika/parser/mbox/MboxParserTest.java | 2 +- .../detect/microsoft/POIFSContainerDetector.java | 2 +- .../apache/tika/parser/microsoft/EMFParser.java | 6 +- .../tika/parser/microsoft/JackcessExtractor.java | 6 +- .../tika/parser/microsoft/JackcessParser.java | 17 +- .../tika/parser/microsoft/MSOwnerFileParser.java | 2 +- .../apache/tika/parser/microsoft/OfficeParser.java | 2 +- .../tika/parser/microsoft/OldExcelParser.java | 2 +- .../tika/parser/microsoft/OutlookExtractor.java | 4 +- .../tika/parser/microsoft/SummaryExtractor.java | 3 +- .../apache/tika/parser/microsoft/TNEFParser.java | 2 +- .../apache/tika/parser/microsoft/WMFParser.java | 2 +- .../microsoft/activemime/ActiveMimeParser.java | 2 +- .../tika/parser/microsoft/chm/ChmCommons.java | 13 +- .../microsoft/chm/ChmDirectoryListingSet.java | 55 +- .../tika/parser/microsoft/chm/ChmExtractor.java | 3 + .../tika/parser/microsoft/chm/ChmItsfHeader.java | 7 +- .../tika/parser/microsoft/chm/ChmItspHeader.java | 3 +- .../tika/parser/microsoft/chm/ChmLzxBlock.java | 8 +- .../parser/microsoft/chm/ChmLzxcControlData.java | 7 +- .../parser/microsoft/chm/ChmLzxcResetTable.java | 14 +- .../tika/parser/microsoft/chm/ChmParser.java | 2 +- .../tika/parser/microsoft/chm/ChmPmgiHeader.java | 3 +- .../tika/parser/microsoft/chm/ChmPmglHeader.java | 3 +- .../tika/parser/microsoft/chm/ChmSection.java | 15 +- .../tika/parser/microsoft/libpst/LibPstParser.java | 2 +- .../microsoft/msg/ExtendedMetadataExtractor.java | 2 +- .../parser/microsoft/onenote/OneNoteParser.java | 2 +- .../tika/parser/microsoft/ooxml/OOXMLParser.java | 2 +- .../ooxml/xwpf/ml2006/Word2006MLParser.java | 2 +- .../parser/microsoft/pst/OutlookPSTParser.java | 2 +- .../parser/microsoft/pst/PSTMailItemParser.java | 2 +- .../parser/microsoft/rtf/RTFObjDataParser.java | 8 +- .../tika/parser/microsoft/rtf/RTFParser.java | 2 +- .../rtf/jflex/RTFObjDataStreamParser.java | 8 +- .../parser/microsoft/xml/SpreadsheetMLParser.java | 2 +- .../tika/parser/microsoft/xml/WordMLParser.java | 2 +- .../tika/parser/microsoft/OutlookParserTest.java | 5 +- .../microsoft/chm/TestChmLzxcControlData.java | 36 ++ .../parser/microsoft/chm/TestChmMalformed.java | 194 ++++++ .../tika/parser/microsoft/chm/TestParameters.java | 3 +- .../rtf/jflex/RTFEmbeddedHandlerTest.java | 2 +- .../apache/tika/detect/ole/MiscOLEDetector.java | 2 +- .../java/org/apache/tika/parser/dbf/DBFParser.java | 2 +- .../java/org/apache/tika/parser/dif/DIFParser.java | 2 +- .../org/apache/tika/parser/epub/EpubParser.java | 2 +- .../org/apache/tika/parser/hwp/HwpV5Parser.java | 2 +- .../apache/tika/parser/indesign/IDMLParser.java | 16 +- .../apache/tika/parser/mif/MIFContentHandler.java | 3 +- .../java/org/apache/tika/parser/mif/MIFParser.java | 2 +- .../tika/parser/odf/FlatOpenDocumentParser.java | 2 +- .../tika/parser/odf/OpenDocumentMetaParser.java | 2 +- .../apache/tika/parser/odf/OpenDocumentParser.java | 2 +- .../tika/parser/wordperfect/QuattroProParser.java | 2 +- .../tika/parser/wordperfect/WordPerfectParser.java | 2 +- .../tika/parser/indesign/IDMLParserTest.java | 5 +- .../org/apache/tika/parser/mif/MIFParserTest.java | 5 +- .../org/apache/tika/parser/feed/FeedParser.java | 2 +- .../apache/tika/parser/iptc/IptcAnpaParser.java | 2 +- .../tika/parser/ocr/{tess4j => }/ImageDeskew.java | 2 +- .../apache/tika/parser/ocr/ImagePreprocessor.java | 1 - .../tika/parser/ocr/{tess4j => }/ImageUtil.java | 2 +- .../apache/tika/parser/ocr/TesseractOCRParser.java | 10 +- .../tika-parser-pdf-module/pom.xml | 5 - .../apache/tika/parser/pdf/AbstractPDF2XHTML.java | 11 +- .../java/org/apache/tika/parser/pdf/PDFParser.java | 11 +- .../tika/parser/pdf/PDMetadataExtractor.java | 440 +------------- .../pdf/xmpschemas/XMPSchemaIllustrator.java | 42 -- .../tika/parser/pdf/xmpschemas/XMPSchemaPDFUA.java | 38 -- .../tika/parser/pdf/xmpschemas/XMPSchemaPDFVT.java | 46 -- .../tika/parser/pdf/xmpschemas/XMPSchemaPDFX.java | 46 -- .../parser/pdf/xmpschemas/XMPSchemaPDFXId.java | 38 -- .../tika/renderer/pdf/pdfbox/PDFBoxRenderer.java | 2 +- .../tika/renderer/pdf/poppler/PopplerRenderer.java | 2 +- .../apache/tika/parser/pdf/CustomTikaXMPTest.java | 19 +- .../org/apache/tika/parser/pdf/PDFParserTest.java | 12 +- .../detect/gzip/GZipSpecializationDetector.java | 2 +- .../apache/tika/parser/pkg/CompressorParser.java | 2 +- .../org/apache/tika/parser/pkg/PackageParser.java | 2 +- .../java/org/apache/tika/parser/pkg/RarParser.java | 2 +- .../org/apache/tika/parser/pkg/SevenZParser.java | 2 +- .../org/apache/tika/parser/pkg/UnrarParser.java | 2 +- .../java/org/apache/tika/parser/pkg/ZipParser.java | 2 +- .../tika-parser-text-module/pom.xml | 17 +- .../apache/tika/parser/csv/TextAndCSVParser.java | 6 +- .../tika/parser/markdown/MarkdownParser.java | 511 ++++++++++++++++ .../tika/parser/strings/Latin1StringsParser.java | 2 +- .../apache/tika/parser/strings/StringsParser.java | 2 +- .../java/org/apache/tika/parser/txt/TXTParser.java | 2 +- .../tika/parser/markdown/MarkdownParserTest.java | 145 +++++ .../test/resources/test-documents/testMARKDOWN.md | 25 + .../test-documents/testMARKDOWN_dataURI.md | 7 + .../org/apache/tika/parser/http/HttpParser.java | 2 +- .../org/apache/tika/parser/wacz/WACZParser.java | 2 +- .../org/apache/tika/parser/warc/WARCParser.java | 12 +- .../apache/tika/parser/warc/WARCParserTest.java | 4 +- .../java/org/apache/tika/parser/tmx/TMXParser.java | 2 +- .../apache/tika/parser/xliff/XLIFF12Parser.java | 2 +- .../org/apache/tika/parser/xliff/XLZParser.java | 2 +- .../org/apache/tika/parser/xml/DcXMLParser.java | 2 +- .../apache/tika/parser/xml/FictionBookParser.java | 2 +- .../java/org/apache/tika/parser/xml/XMLParser.java | 2 +- .../org/apache/tika/parser/xml/XMLProfiler.java | 8 +- .../tika-parser-xmp-commons/pom.xml | 10 +- .../apache/tika/parser/xmp/JempboxExtractor.java | 246 -------- .../tika/parser/xmp/XMPMetadataExtractor.java | 264 --------- .../apache/tika/parser/xmp/XMPPacketScanner.java | 8 + .../java/org/apache/tika/parser/xmp/XmpDates.java | 60 ++ .../org/apache/tika/parser/xmp/XmpExtractor.java | 454 ++++++++++++++ .../org/apache/tika/parser/xmp/XmpProperty.java | 51 ++ .../apache/tika/parser/xmp/XmpSaxFlattener.java | 231 ++++++++ .../tika/parser/xmp/JempboxExtractorTest.java | 157 ----- .../org/apache/tika/parser/xmp/XmpDatesTest.java | 74 +++ .../apache/tika/parser/xmp/XmpExtractorTest.java | 483 +++++++++++++++ .../tika/parser/xmp/XmpSaxFlattenerTest.java | 155 +++++ .../tika/parser/xmp/XmpboxExtractorTest.java | 165 ------ .../detect/zip/DefaultZipContainerDetector.java | 2 +- .../apache/tika/async/cli/AsyncProcessorTest.java | 2 +- .../java/org/apache/tika/pipes/api/ParseMode.java | 4 +- .../org/apache/tika/pipes/api/PipesResult.java | 1 + .../org/apache/tika/pipes/core/PipesClient.java | 17 +- .../org/apache/tika/pipes/core/PipesConfig.java | 32 + .../core/extractor/StandardUnpackSelector.java | 2 +- .../tika/pipes/core/extractor/UnpackConfig.java | 2 +- .../core/extractor/UnpackExtractorFactory.java | 2 +- .../protocol/PayloadLimitExceededException.java} | 28 +- .../tika/pipes/core/protocol/PipesMessage.java | 27 +- .../serialization/FetchEmitTupleDeserializer.java | 4 +- .../tika/pipes/core/server/ConnectionHandler.java | 2 +- .../apache/tika/pipes/core/server/EmitHandler.java | 17 +- .../apache/tika/pipes/core/server/PipesServer.java | 4 +- .../tika/pipes/core/TikaPipesConfigTest.java | 52 ++ .../tika/pipes/core/protocol/PipesMessageTest.java | 33 +- .../filter/AttachmentCountingListFilter.java | 6 +- .../tika/metadata/filter/MockUpperCaseFilter.java | 2 +- .../apache/tika/pipes/core/CrashingDetector.java | 2 +- .../pipes/core/DigestingOpenContainersTest.java | 2 +- .../tika/pipes/core/MetadataWriteLimiterTest.java | 22 +- .../apache/tika/pipes/core/MockPassbackFilter.java | 2 +- .../apache/tika/pipes/core/PipesClientTest.java | 6 +- .../apache/tika/pipes/core/PipesServerTest.java | 2 +- .../apache/tika/pipes/core/PluginsTestHelper.java | 2 +- .../pipes/core/async/AsyncChaosMonkeyTest.java | 2 +- .../pipes/core/emitter/EmitterManagerTest.java | 2 +- .../pipes/core/fetcher/FetcherManagerTest.java | 2 +- .../configs/tika-config-write-limiter.json | 2 +- .../tika/pipes/emitter/azblob/AZBlobEmitter.java | 2 +- .../org/apache/tika/pipes/emitter/es/ESClient.java | 2 +- .../apache/tika/pipes/emitter/es/ESClientTest.java | 24 +- .../tika-pipes-google-drive/pom.xml | 6 +- .../pipes/emitter/jdbc/JDBCEmitterFactory.java | 2 +- .../tika-pipes-microsoft-graph/pom.xml | 5 +- .../pipes/emitter/opensearch/OpenSearchClient.java | 2 +- .../org/apache/tika/config/loader/TikaLoader.java | 2 +- .../{ => serialization}/config/GlobalSettings.java | 2 +- .../config/JsonConfigHelper.java | 2 +- .../serialization/serdes/MetadataDeserializer.java | 4 +- .../apache/tika/config/JsonConfigHelperTest.java | 3 +- .../tika/config/loader/ConfigurableTestParser.java | 2 +- .../tika/config/loader/FallbackTestParser.java | 2 +- .../tika/config/loader/MinimalTestParser.java | 2 +- .../apache/tika/config/loader/OptInTestParser.java | 2 +- .../apache/tika/digest/MockDigesterFactory.java | 4 +- .../filter/AttachmentCountingListFilter.java | 6 +- .../tika/metadata/filter/JsonConfigOnlyFilter.java | 2 +- .../tika/metadata/filter/MockUpperCaseFilter.java | 2 +- .../writefilter/StandardMetadataLimiterTest.java | 7 +- .../tika/sax/UppercasingContentHandlerFactory.java | 2 +- .../CustomClassSerializationTest.java | 2 +- .../tika/serialization/JsonMetadataListTest.java | 2 +- .../tika/serialization/JsonMetadataTest.java | 24 +- .../resources/configs/TIKA-3137-field-mapping.json | 2 +- .../test/resources/configs/TIKA-3695-fields.json | 2 +- .../src/test/resources/configs/TIKA-3695.json | 2 +- tika-server/docker-build/full/Dockerfile | 15 +- tika-server/docker-build/full/Dockerfile.snapshot | 14 +- tika-server/docker-build/minimal/Dockerfile | 14 +- .../docker-build/minimal/Dockerfile.snapshot | 14 +- .../apache/tika/server/core/TikaServerConfig.java | 1 + .../apache/tika/server/core/TikaServerProcess.java | 9 +- .../server/core/resource/PipesParsingHelper.java | 1 + .../tika/server/core/resource/TikaResource.java | 6 +- .../org/apache/tika/server/core/CXFTestBase.java | 2 +- .../apache/tika/server/core/StackTraceTest.java | 2 +- .../tika/server/core/TikaServerConfigTest.java | 72 +++ .../server/core/TikaServerIntegrationTest.java | 8 +- .../server/core/TikaServerProcessTlsGuardTest.java | 82 +++ .../resources/configs/metadata-filter-include.json | 2 +- ...onfig-server-tls-client-auth-no-truststore.json | 15 + .../tika-config-server-tls-client-auth-valid.json | 18 + .../tika-config-server-tls-missing-keystore.json | 14 + .../tika-config-server-tls-partial-truststore.json | 15 + .../tika/server/standard/MetadataResourceTest.java | 2 +- .../server/standard/OpenNLPMetadataFilterTest.java | 4 +- .../standard/OptimaizeMetadataFilterTest.java | 4 +- .../standard/RecursiveMetadataFilterTest.java | 2 +- .../standard/RecursiveMetadataResourceTest.java | 10 +- .../tika/server/standard/UnpackerResourceTest.java | 12 +- .../standard/UnpackerResourceWithConfigTest.java | 2 +- .../test/resources/configs/TIKA-3137-include.json | 2 +- tika-translate/pom.xml | 6 - .../language/translate/impl/CachedTranslator.java | 2 +- .../language/translate/impl/GoogleTranslator.java | 2 +- .../translate/impl/JoshuaNetworkTranslator.java | 2 +- .../language/translate/impl/Lingo24Translator.java | 2 +- .../language/translate/impl/MarianTranslator.java | 2 +- .../translate/impl/MicrosoftTranslator.java | 2 +- .../language/translate/impl/MosesTranslator.java | 2 +- .../language/translate/impl/RTGTranslator.java | 2 +- .../language/translate/impl/YandexTranslator.java | 2 +- .../tika/xmp/convert/MSOfficeBinaryConverter.java | 2 +- .../tika/xmp/convert/MSOfficeXMLConverter.java | 2 +- .../tika/xmp/convert/OpenDocumentConverter.java | 2 +- .../org/apache/tika/xmp/convert/RTFConverter.java | 2 +- 671 files changed, 13474 insertions(+), 3130 deletions(-)
