This is an automated email from the ASF dual-hosted git repository. nddipiazza pushed a commit to branch pr-3018 in repository https://gitbox.apache.org/repos/asf/tika.git
commit 6f07d05d0fd8fcba9df8fc844283f008ee45d568 Merge: 177d21b9da 7d6fe60962 Author: nddipiazza <[email protected]> AuthorDate: Thu Aug 13 17:38:30 2026 -0500 Merge remote-tracking branch 'origin/main' into pr-3018 .devcontainer/Dockerfile | 22 + .../index.adoc => .devcontainer/devcontainer.json | 26 +- .github/workflows/main-jdk17-build.yml | 5 +- .skills/dev.md | 133 -- .skills/dev/SKILL.md | 210 +++ .../SKILL.md} | 34 +- .skills/pr-review/SKILL.md | 211 +++ .../SKILL.md} | 31 +- .../SKILL.md} | 25 + .../SKILL.md} | 25 + AGENTS.md | 34 + CHANGES.txt | 117 ++ .../integration-testing/run-uat-script.adoc | 14 +- .../advanced/integration-testing/tika-app.adoc | 2 +- .../integration-testing/tika-eval-regression.adoc | 3 +- .../advanced/integration-testing/tika-server.adoc | 26 +- .../ROOT/pages/advanced/local-vlm-server.adoc | 6 +- .../ROOT/pages/advanced/setting-limits.adoc | 68 +- .../configuration/parsers/external-parser.adoc | 2 +- .../pages/configuration/parsers/tess4j-parser.adoc | 9 +- .../pages/configuration/parsers/vlm-parsers.adoc | 6 +- docs/modules/ROOT/pages/developers/index.adoc | 2 +- .../ROOT/pages/developers/metadata-keys.adoc | 43 +- .../ROOT/pages/developers/serialization.adoc | 19 +- .../pages/migration-to-4x/metadata-changes-4x.adoc | 226 +++- .../migration-to-4x/migrating-tika-server-4x.adoc | 229 +++- .../pages/migration-to-4x/migrating-to-4x.adoc | 44 +- docs/modules/ROOT/pages/pipes/configuration.adoc | 24 +- docs/modules/ROOT/pages/pipes/cpu-sizing.adoc | 33 +- docs/modules/ROOT/pages/pipes/getting-started.adoc | 9 +- docs/modules/ROOT/pages/pipes/plugins/http.adoc | 47 +- docs/modules/ROOT/pages/pipes/timeouts.adoc | 485 +++++-- docs/modules/ROOT/pages/pipes/troubleshooting.adoc | 2 +- docs/modules/ROOT/pages/using-tika/cli/index.adoc | 16 +- .../ROOT/pages/using-tika/server/index.adoc | 150 +- llms.txt | 40 + pom.xml | 2 + release-tools/uat/run-uat.sh | 51 +- .../src/main/java/org/apache/tika/cli/TikaCLI.java | 37 +- .../apache/tika/cli/XmlToJsonConfigConverter.java | 21 +- .../src/main/java/org/apache/tika/gui/TikaGUI.java | 3 +- .../test/java/org/apache/tika/cli/TikaCLITest.java | 39 +- .../tika/cli/XmlToJsonConfigConverterTest.java | 32 +- .../test/resources/configs/config-template.json | 2 - .../xml-configs/tika-config-list-map-types.xml | 2 +- tika-bom/pom.xml | 6 - tika-bundles/tika-bundle-standard/pom.xml | 2 +- .../org/apache/tika/config/EmbeddedLimits.java | 4 +- .../java/org/apache/tika/config/OutputLimits.java | 2 +- .../java/org/apache/tika/config/ParseTimeout.java | 246 ++++ .../apache/tika/config/TikaProgressTracker.java | 82 -- .../java/org/apache/tika/config/TimeoutLimits.java | 127 +- .../apache/tika/detect/FileCommandDetector.java | 20 +- .../tika/detect/MetadataCharsetDetector.java | 15 +- .../java/org/apache/tika/detect/TypeDetector.java | 3 +- .../apache/tika/digest/InputStreamDigester.java | 21 +- .../exception/EmbeddedLimitReachedException.java | 22 +- .../tika/exception/TikaTimeoutException.java | 71 +- .../tika/extractor/EmbeddedDocumentUtil.java | 5 +- .../ParsingEmbeddedDocumentExtractor.java | 45 +- .../java/org/apache/tika/io/CachingSource.java | 5 +- .../java/org/apache/tika/io/FilenameUtils.java | 5 +- .../java/org/apache/tika/io/TikaInputStream.java | 17 +- .../main/java/org/apache/tika/metadata/Audio.java | 39 + .../org/apache/tika/metadata/ClimateForcast.java | 55 - .../org/apache/tika/metadata/ClimateForecast.java | 63 + .../main/java/org/apache/tika/metadata/DWG.java | 4 +- .../main/java/org/apache/tika/metadata/Epub.java | 2 +- .../main/java/org/apache/tika/metadata/HTML.java | 2 +- .../tika/metadata/{TikaMimeKeys.java => IDML.java} | 16 +- .../main/java/org/apache/tika/metadata/IPTC.java | 91 +- .../java/org/apache/tika/metadata/ISO19115.java | 122 ++ .../java/org/apache/tika/metadata/KeyPrefix.java | 182 +++ .../main/java/org/apache/tika/metadata/MAPI.java | 4 +- .../apache/tika/metadata/{Epub.java => MIF.java} | 24 +- .../org/apache/tika/metadata/MachineMetadata.java | 8 +- .../java/org/apache/tika/metadata/Message.java | 2 +- .../java/org/apache/tika/metadata/Metadata.java | 328 ++++- .../main/java/org/apache/tika/metadata/Office.java | 2 +- .../apache/tika/metadata/OfficeOpenXMLCore.java | 7 - .../tika/metadata/OfficeOpenXMLExtended.java | 2 +- .../java/org/apache/tika/metadata/OneNote.java | 63 + .../main/java/org/apache/tika/metadata/PDF.java | 7 +- .../apache/tika/metadata/PassthroughPrefix.java | 79 -- .../java/org/apache/tika/metadata/Photoshop.java | 2 +- .../java/org/apache/tika/metadata/Property.java | 227 +++- .../java/org/apache/tika/metadata/RTFMetadata.java | 2 +- .../java/org/apache/tika/metadata/Rendering.java | 18 +- .../apache/tika/metadata/ReservedNamespaces.java | 10 +- .../main/java/org/apache/tika/metadata/TIFF.java | 4 +- .../apache/tika/metadata/TikaCoreProperties.java | 156 ++- .../org/apache/tika/metadata/TikaPagedText.java | 6 +- .../main/java/org/apache/tika/metadata/XMPDM.java | 84 +- .../main/java/org/apache/tika/metadata/XMPMM.java | 2 +- .../main/java/org/apache/tika/metadata/Zip.java | 2 +- .../metadata/filter/LegacyKeyMigrationFilter.java | 175 ++- .../filter/RemoveByMimeMetadataFilter.java | 3 +- .../MetadataWriteLimiter.java | 2 +- .../MetadataWriteLimiterFactory.java | 4 +- .../StandardMetadataLimiter.java | 56 +- .../StandardMetadataLimiterFactory.java | 2 +- .../main/java/org/apache/tika/mime/MimeTypes.java | 3 +- .../mime/ProbabilisticMimeDetectionSelector.java | 3 +- .../org/apache/tika/parser/AutoDetectParser.java | 3 +- .../org/apache/tika/parser/CompositeParser.java | 19 +- .../java/org/apache/tika/parser/NetworkParser.java | 40 +- .../java/org/apache/tika/parser/ParseContext.java | 2 +- .../java/org/apache/tika/parser/ParseRecord.java | 30 + .../apache/tika/parser/RecursiveParserWrapper.java | 3 +- .../tika/parser/external/ExternalParser.java | 13 +- .../tika/parser/external/ExternalParserConfig.java | 10 +- .../parser/multiple/AbstractMultipleParser.java | 3 +- .../sax/AbstractRecursiveParserWrapperHandler.java | 6 - .../tika/sax/BasicContentHandlerFactory.java | 20 +- .../org/apache/tika/sax/DIFContentHandler.java | 150 -- .../tika/sax/RecursiveParserWrapperHandler.java | 2 - .../org/apache/tika/sax/XHTMLContentHandler.java | 3 +- .../java/org/apache/tika/utils/ExceptionUtils.java | 22 - .../org/apache/tika/utils/FileProcessResult.java | 39 + .../java/org/apache/tika/utils/ParserUtils.java | 18 +- .../java/org/apache/tika/utils/ProcessUtils.java | 167 ++- .../tika/metadata/metadata-migration-3x-4x.json | 115 +- .../org/apache/tika/config/ParseTimeoutTest.java | 267 ++++ .../tika/config/TikaProgressTrackerTest.java | 103 -- .../org/apache/tika/config/TimeoutLimitsTest.java | 47 + .../tika/detect/FileCommandDetectorTest.java | 26 + .../tika/detect/MetadataCharsetDetectorTest.java | 13 +- .../org/apache/tika/detect/TypeDetectorTest.java | 3 +- .../EmbeddedDocumentUtilExtensionTest.java | 9 +- ...arsingEmbeddedDocumentExtractorTimeoutTest.java | 277 ++++ .../java/org/apache/tika/io/FilenameUtilsTest.java | 3 +- .../org/apache/tika/io/TikaInputStreamTest.java | 11 +- .../org/apache/tika/metadata/KeyPrefixTest.java | 126 ++ .../metadata/MetadataInternalKeyGuardTest.java | 96 +- .../tika/metadata/MetadataKeyPrefixRouteTest.java | 130 ++ .../apache/tika/metadata/MetadataPutAllTest.java | 141 ++ .../tika/metadata/MetadataTypedAccessorsTest.java | 117 ++ .../tika/metadata/PropertyReservedNameTest.java | 161 +++ .../org/apache/tika/metadata/PropertyTest.java | 113 ++ .../org/apache/tika/metadata/TestMetadata.java | 61 +- .../filter/LegacyKeyMigrationFilterTest.java | 131 ++ .../org/apache/tika/mime/MimeDetectionTest.java | 3 +- .../tika/mime/ProbabilisticMimeDetectionTest.java | 5 +- .../ProbabilisticMimeDetectionTestWithTika.java | 3 +- .../apache/tika/parser/CompositeParserTest.java | 61 +- .../org/apache/tika/parser/NetworkParserTest.java | 135 ++ .../org/apache/tika/parser/mock/MockParser.java | 28 +- .../tika/parser/multiple/MultipleParserTest.java | 5 +- .../tika/sax/BasicContentHandlerFactoryTest.java | 33 + .../org/apache/tika/utils/ProcessUtilsTest.java | 136 ++ .../apache/tika/detect/magika/MagikaDetector.java | 18 +- .../tika/detect/magika/TestMagikaIntegration.java | 3 +- .../tika/detect/siegfried/SiegfriedDetector.java | 18 +- .../resources/tika-config-ignite-handlertype.json | 8 +- .../test/resources/tika-config-ignite-local.json | 7 +- .../src/test/resources/tika-config-ignite.json | 7 +- .../tika-grpc/src/test/resources/tika-config.json | 7 +- .../apache/tika/server/e2e/RunUatSmokeTest.java | 6 +- .../tika/server/e2e/TikaServerHttp2Test.java | 6 +- .../html/StandardHtmlEncodingDetectorTest.java | 5 +- .../tika/detect/icu4j/Icu4jEncodingDetector.java | 5 +- .../ml/chardetect/MojibusterEncodingDetector.java | 3 +- .../universal/UniversalEncodingListener.java | 5 +- .../org/apache/tika/eval/app/ProfilerBase.java | 16 +- .../org/apache/tika/eval/app/io/ExtractReader.java | 15 +- .../apache/tika/eval/app/io/ExtractReaderTest.java | 3 +- .../src/test/resources/extracts/TIKA-4446-a.json | 2 +- .../src/test/resources/extracts/TIKA-4446-b.json | 2 +- .../resources/test-dirs/extractsA/file15_tags.json | 2 +- .../test-dirs/extractsA/file16_badTags.json | 2 +- .../test-dirs/extractsA/file17_tagsOutOfOrder.json | 2 +- .../test/resources/test-dirs/raw_input/file15_tags | 2 +- .../resources/test-dirs/raw_input/file16_badTags | 2 +- .../test-dirs/raw_input/file17_tagsOutOfOrder | 2 +- .../tika/eval/core/util/EvalExceptionUtils.java | 16 +- .../org/apache/tika/example/ImportContextImpl.java | 3 +- .../tika/example/MetadataAwareLuceneIndexer.java | 2 +- .../java/org/apache/tika/example/MyFirstTika.java | 3 +- .../tika/example/PipesForkParserExample.java | 11 +- .../org/apache/tika/example/TIAParsingExample.java | 3 +- .../apache/tika/pipes/grpc/TikaGrpcServerImpl.java | 1 + .../apache/tika/pipes/grpc/TikaGrpcServerTest.java | 71 + .../src/test/resources/tika-pipes-test-config.json | 4 +- tika-metadata-schema/README.md | 29 +- tika-metadata-schema/pom.xml | 37 +- tika-metadata-schema/regen.sh | 34 +- .../tika/metadata/schema/MetadataKeyValidator.java | 81 +- .../metadata/schema/MigrationTableGenerator.java | 95 +- .../tika/metadata/schema/SchemaGenerator.java | 140 +- .../apache/tika/metadata/metadata-key-fields.json | 142 +- .../org/apache/tika/metadata/metadata-keys.json | 1426 +++++++++++--------- .../tika/metadata/metadata-open-namespaces.json | 61 +- .../schema/MetadataChangesAdocDriftTest.java | 122 ++ .../tika/metadata/schema/MetadataCoverageTest.java | 16 +- .../metadata/schema/MetadataKeyValidatorTest.java | 10 + .../schema/MetadataMigrationTableTest.java | 4 +- .../metadata/schema/MetadataNoUnderscoreTest.java | 18 +- .../tika/metadata/schema/MetadataSchemaTest.java | 4 +- .../metadata/schema/MigrationCompletenessTest.java | 119 ++ .../org/apache/tika/metadata/migration-overlay.tsv | 223 ++- .../forbidden-apis-metadata-string-key-bans.txt | 32 + tika-parent/pom.xml | 19 +- .../java/org/apache/tika/http/TikaHttpClient.java | 185 ++- .../org/apache/tika/http/TikaHttpClientTest.java | 185 +++ .../org/apache/tika/http/TikaTestHttpServer.java | 38 +- .../tika/parser/ocrencode/EncodeOCRParser.java | 5 +- .../tika-parser-scientific-module/pom.xml | 15 + .../tika/parser/climate/ClimateForecast.java | 64 + .../apache/tika/parser/envi/EnviHeaderParser.java | 19 +- .../org/apache/tika/parser/gdal/GDALParser.java | 86 +- .../geoinfo/GeographicInformationParser.java | 101 +- .../org/apache/tika/parser/grib/GribParser.java | 33 +- .../java/org/apache/tika/parser/hdf/HDFParser.java | 17 +- .../org/apache/tika/parser/isatab/ISATabUtils.java | 14 +- .../apache/tika/parser/isatab/ISArchiveParser.java | 9 +- .../apache/tika/parser/netcdf/NetCDFParser.java | 35 +- .../apache/tika/parser/gdal/TestGDALParser.java | 70 +- .../geoinfo/GeographicInformationParserTest.java | 12 +- .../org/apache/tika/parser/hdf/HDFParserTest.java | 8 +- .../tika/parser/isatab/ISArchiveParserTest.java | 15 +- .../tika/parser/netcdf/NetCDFParserTest.java | 13 +- .../tika/inference/AbstractEmbeddingFilter.java | 23 +- .../tika/inference/ImageEmbeddingConfig.java | 10 +- .../org/apache/tika/inference/InferenceConfig.java | 12 +- .../tika/inference/OpenAIEmbeddingFilter.java | 7 +- .../tika/inference/OpenAIImageEmbeddingParser.java | 26 +- .../tika/inference/JinaEmbeddingFilterTest.java | 2 +- .../tika/inference/OpenAIEmbeddingFilterTest.java | 2 +- .../inference/OpenAIImageEmbeddingParserTest.java | 21 +- .../tika-parsers-ml/tika-parser-nlp-module/pom.xml | 15 + .../tika/parser/ctakes/CTAKESContentHandler.java | 10 +- .../apache/tika/parser/geo/topic/GeoParser.java | 31 +- .../tika/parser/journal/GrobidRESTParser.java | 21 +- .../apache/tika/parser/journal/TEIDOMParser.java | 41 +- .../apache/tika/parser/ner/NamedEntityParser.java | 18 +- .../tika/parser/geo/topic/GeoParserTest.java | 24 +- .../tika/parser/journal/JournalParserTest.java | 2 +- .../org/apache/tika/parser/journal/TEITest.java | 13 +- .../tika/parser/ner/NamedEntityParserTest.java | 12 +- .../tika/parser/ner/nltk/NLTKNERecogniserTest.java | 2 +- .../parser/ner/regex/RegexNERecogniserTest.java | 2 +- .../tika/parser/ocr/tess4j/Tess4JConfig.java | 19 +- .../tika/parser/ocr/tess4j/Tess4JParser.java | 155 ++- .../tika/parser/ocr/tess4j/Tess4JConfigTest.java | 2 +- .../parser/ocr/tess4j/Tess4JOCRTimeoutTest.java | 250 ++++ .../tika/parser/ocr/tess4j/Tess4JParserTest.java | 6 +- .../resources/config-examples/tess4j-full.json | 2 +- .../apache/tika/parser/vlm/AbstractVLMParser.java | 24 +- .../org/apache/tika/parser/vlm/VLMOCRConfig.java | 12 +- .../tika/parser/vlm/ClaudeVLMParserTest.java | 13 +- .../tika/parser/vlm/GeminiVLMParserTest.java | 13 +- .../tika/parser/vlm/OpenAIVLMParserTest.java | 19 +- .../resources/config-examples/claude-vlm-full.json | 2 +- .../resources/config-examples/gemini-vlm-full.json | 2 +- .../config-examples/openai-vlm-basic.json | 2 +- .../resources/config-examples/openai-vlm-full.json | 2 +- .../tika/detect/TestContainerAwareDetector.java | 3 +- .../java/org/apache/tika/mime/OneOffMimeTest.java | 5 +- .../apache/tika/parser/AndroidBinaryXMLTest.java | 7 +- .../tika/parser/AutoDetectParserConfigTest.java | 3 +- .../apache/tika/parser/AutoDetectParserTest.java | 11 +- .../tika/parser/AutoDetectReaderParserTest.java | 3 +- .../tika/parser/RecursiveParserWrapperTest.java | 7 +- .../apache/tika/parser/apple/PListParserTest.java | 3 +- .../tika/parser/crypto/Pkcs7EndToEndTest.java | 7 +- .../apache/tika/parser/crypto/TSDParserTest.java | 3 +- .../tika/parser/digest/DigestConfigTest.java | 5 +- .../parser/html/HtmlEncodingDetectionTest.java | 11 +- .../apache/tika/parser/html/HtmlParserTest.java | 5 +- .../apache/tika/parser/image/JpegParserTest.java | 5 +- .../apache/tika/parser/mail/MboxParserTest.java | 3 +- .../apache/tika/parser/mail/RFC822ParserTest.java | 10 +- .../tika/parser/microsoft/EMFParserTest.java | 7 +- .../tika/parser/microsoft/XML2003ParserTest.java | 3 +- .../microsoft/ooxml/OleObjectRegressionTest.java | 5 +- .../parser/microsoft/ooxml/TruncatedOOXMLTest.java | 5 +- .../tika/parser/microsoft/rtf/RTFParserTest.java | 3 +- .../tika/parser/ocr/TesseractOCRParserTest.java | 21 +- .../org/apache/tika/parser/odf/ODFParserTest.java | 21 +- .../org/apache/tika/parser/pdf/PDFParserTest.java | 16 +- .../org/apache/tika/parser/pkg/ArParserTest.java | 5 +- .../apache/tika/parser/pkg/Bzip2ParserTest.java | 3 +- .../apache/tika/parser/pkg/CompressParserTest.java | 3 +- .../tika/parser/pkg/CompressorParserTest.java | 5 +- .../org/apache/tika/parser/pkg/GzipParserTest.java | 5 +- .../org/apache/tika/parser/pkg/RarParserTest.java | 3 +- .../apache/tika/parser/pkg/Seven7ParserTest.java | 5 +- .../org/apache/tika/parser/pkg/TarParserTest.java | 3 +- .../org/apache/tika/parser/pkg/ZipParserTest.java | 3 +- .../org/apache/tika/parser/pkg/ZlibParserTest.java | 3 +- .../config-examples/migration-full-example.json | 2 +- .../resources/config-examples/tesseract-basic.json | 2 +- .../resources/config-examples/tesseract-full.json | 2 +- .../tika-parser-apple-module/pom.xml | 15 + .../org/apache/tika/parser/apple/PListParser.java | 5 +- .../tika/parser/iwork/IWorkPackageParser.java | 3 +- .../tika/parser/iwork/KeynoteContentHandler.java | 8 +- .../tika/parser/iwork/NumbersContentHandler.java | 14 +- .../tika/parser/iwork/PagesContentHandler.java | 36 +- .../parser/iwork/iwana/IWork13PackageParser.java | 13 +- .../parser/iwork/iwana/IWork18PackageParser.java | 3 +- .../apache/tika/parser/apple/PListParserTest.java | 3 +- .../apache/tika/parser/iwork/IWorkParserTest.java | 16 +- .../tika/parser/iwork/iwana/IWork13ParserTest.java | 15 +- .../tika-parser-audiovideo-module/pom.xml | 15 + .../org/apache/tika/parser/audio/AudioParser.java | 59 +- .../org/apache/tika/parser/audio/MidiParser.java | 25 +- .../org/apache/tika/parser/mp3/ID3v2Frame.java | 16 +- .../java/org/apache/tika/parser/mp3/Mp3Parser.java | 15 +- .../java/org/apache/tika/parser/mp4/MP4Parser.java | 44 +- .../apache/tika/parser/mp4/TikaMp4BoxHandler.java | 7 +- .../org/apache/tika/parser/mp4/TikaMp4Reader.java | 149 ++ .../tika/parser/mp4/TikaMp4SoundHandler.java | 15 +- .../tika/parser/mp4/boxes/TikaUserDataBox.java | 174 ++- .../org/apache/tika/parser/ogg/FlacParser.java | 10 +- .../org/apache/tika/parser/ogg/OggAudioParser.java | 27 +- .../java/org/apache/tika/parser/ogg/OggParser.java | 12 +- .../org/apache/tika/parser/ogg/OpusParser.java | 6 +- .../org/apache/tika/parser/ogg/SpeexParser.java | 6 +- .../org/apache/tika/parser/ogg/TheoraParser.java | 5 +- .../org/apache/tika/parser/ogg/VorbisParser.java | 5 +- .../org/apache/tika/parser/video/FLVParser.java | 71 +- .../apache/tika/parser/audio/AudioParserTest.java | 57 +- .../apache/tika/parser/audio/MidiParserTest.java | 9 +- .../org/apache/tika/parser/mp3/ID3v2FrameTest.java | 13 + .../org/apache/tika/parser/mp3/Mp3ParserTest.java | 99 +- .../org/apache/tika/parser/mp4/MP4ParserTest.java | 150 +- .../tika/parser/mp4/boxes/TikaUserDataBoxTest.java | 61 + .../org/apache/tika/parser/ogg/FlacParserTest.java | 9 +- .../apache/tika/parser/ogg/OggAudioParserTest.java | 3 +- .../apache/tika/parser/ogg/VorbisParserTest.java | 9 +- .../apache/tika/parser/video/FLVParserTest.java | 25 +- .../tika-parser-cad-module/pom.xml | 15 + .../java/org/apache/tika/parser/dwg/DWGParser.java | 19 +- .../apache/tika/parser/dwg/DWGParserConfig.java | 10 +- .../org/apache/tika/parser/dwg/DWGReadParser.java | 13 +- .../java/org/apache/tika/parser/prt/PRTParser.java | 3 +- .../org/apache/tika/parser/dwg/DWGParserTest.java | 14 +- .../org/apache/tika/parser/prt/PRTParserTest.java | 5 +- .../configs/tika-config-dwgRead-Timeout.json | 2 +- .../apache/tika/parser/code/SourceCodeParser.java | 7 +- .../tika/parser/executable/ExecutableParser.java | 37 +- .../java/org/apache/tika/parser/mat/MatParser.java | 3 +- .../org/apache/tika/parser/sas/SAS7BDATParser.java | 2 +- .../tika/parser/code/SourceCodeParserTest.java | 3 +- .../parser/executable/ExecutableParserTest.java | 9 +- .../executable/UniversalExecutableParserTest.java | 5 +- .../apache/tika/parser/sas/SAS7BDATParserTest.java | 4 +- .../org/apache/tika/parser/crypto/Pkcs7Parser.java | 7 +- .../org/apache/tika/parser/crypto/TSDParser.java | 3 +- .../apache/tika/parser/crypto/Pkcs7ParserTest.java | 3 +- .../tika-parser-font-module/pom.xml | 15 + .../tika/parser/font/AdobeFontMetricParser.java | 6 +- .../apache/tika/parser/font/TrueTypeParser.java | 18 +- .../apache/tika/parser/font/FontParsersTest.java | 5 +- .../org/apache/tika/parser/html/HtmlHandler.java | 28 +- .../org/apache/tika/parser/html/JSoupParser.java | 7 +- .../apache/tika/parser/html/HtmlParserTest.java | 47 +- .../tika-parser-image-module/pom.xml | 15 + .../tika/parser/image/AbstractImageParser.java | 9 +- .../org/apache/tika/parser/image/ICNSParser.java | 12 +- .../tika/parser/image/ImageMetadataExtractor.java | 82 +- .../org/apache/tika/parser/image/ImageParser.java | 23 +- .../org/apache/tika/parser/image/HeicXmpTest.java | 5 +- .../apache/tika/parser/image/HeifParserTest.java | 7 +- .../apache/tika/parser/image/ICNSParserTest.java | 5 +- .../apache/tika/parser/image/ImageParserTest.java | 34 +- .../apache/tika/parser/image/JpegParserTest.java | 133 +- .../tika/parser/image/MotionPhotoXmpTest.java | 5 +- .../apache/tika/parser/image/PSDParserTest.java | 20 +- .../apache/tika/parser/image/WebPParserTest.java | 5 +- .../org/apache/tika/parser/image/WebPXmpTest.java | 5 +- .../apache/tika/parser/jdbc/JDBCTableReader.java | 5 +- .../tika-parser-mail-module/pom.xml | 15 + .../tika/parser/mail/MailContentHandler.java | 29 +- .../org/apache/tika/parser/mbox/MboxParser.java | 34 +- .../apache/tika/parser/mail/RFC822ParserTest.java | 67 +- .../apache/tika/parser/mbox/MboxParserTest.java | 28 +- .../tika-parser-microsoft-module/pom.xml | 15 + .../microsoft/PSTEmailStreamTranslator.java | 3 +- .../parser/microsoft/AbstractPOIFSExtractor.java | 15 +- .../apache/tika/parser/microsoft/EMFParser.java | 3 +- .../tika/parser/microsoft/JackcessExtractor.java | 9 +- .../tika/parser/microsoft/JackcessParser.java | 14 +- .../apache/tika/parser/microsoft/OfficeParser.java | 7 +- .../tika/parser/microsoft/OutlookExtractor.java | 12 +- .../tika/parser/microsoft/SummaryExtractor.java | 36 +- .../apache/tika/parser/microsoft/TNEFParser.java | 3 +- .../tika/parser/microsoft/chm/ChmParser.java | 3 +- .../tika/parser/microsoft/libpst/LibPstParser.java | 7 +- .../microsoft/libpst/LibPstParserConfig.java | 10 +- .../microsoft/msg/ExtendedMetadataExtractor.java | 15 +- .../parser/microsoft/onenote/OneNoteParser.java | 44 +- .../onenote/fsshttpb/MSOneStorePackage.java | 91 +- .../onenote/fsshttpb/MSOneStoreParser.java | 1 + .../streamobj/RevisionStoreObjectGroup.java | 3 + .../microsoft/ooxml/AbstractOOXMLExtractor.java | 5 +- .../microsoft/ooxml/OOXMLExtractorFactory.java | 5 +- .../microsoft/ooxml/SAXBasedMetadataExtractor.java | 50 +- .../parser/microsoft/pst/OutlookPSTParser.java | 7 +- .../parser/microsoft/pst/PSTMailItemParser.java | 7 +- .../parser/microsoft/rtf/RTFEmbObjHandler.java | 8 +- .../parser/microsoft/rtf/RTFObjDataParser.java | 3 +- .../tika/parser/microsoft/rtf/RTFParser.java | 3 +- .../microsoft/rtf/jflex/RTFEmbeddedHandler.java | 8 +- .../rtf/jflex/RTFObjDataStreamParser.java | 3 +- .../parser/microsoft/xml/SpreadsheetMLParser.java | 3 +- .../tika/parser/microsoft/xml/WordMLParser.java | 3 +- .../tika/parser/microsoft/EMFParserTest.java | 3 +- .../tika/parser/microsoft/ExcelParserTest.java | 82 +- .../tika/parser/microsoft/OldExcelParserTest.java | 3 +- .../tika/parser/microsoft/OutlookParserTest.java | 15 +- .../microsoft/POIContainerExtractionTest.java | 7 +- .../parser/microsoft/PowerPointParserTest.java | 13 +- .../tika/parser/microsoft/ProjectParserTest.java | 3 +- .../tika/parser/microsoft/PublisherParserTest.java | 3 +- .../parser/microsoft/SolidworksParserTest.java | 13 +- .../tika/parser/microsoft/VisioParserTest.java | 3 +- .../tika/parser/microsoft/WMFParserTest.java | 3 +- .../tika/parser/microsoft/WordParserTest.java | 21 +- .../microsoft/activemime/ActiveMimeParserTest.java | 3 +- .../parser/microsoft/libpst/TestLibPstParser.java | 3 +- .../onenote/fsshttpb/MSOneStorePackageTest.java | 1 - .../ooxml/OOXMLContainerExtractionTest.java | 3 +- .../parser/microsoft/ooxml/OOXMLDocxSAXTest.java | 21 +- .../parser/microsoft/ooxml/OOXMLParserTest.java | 25 +- .../parser/microsoft/ooxml/OOXMLPptxSAXTest.java | 25 +- .../parser/microsoft/ooxml/TruncatedOOXMLTest.java | 5 +- .../parser/microsoft/ooxml/VSDXParserTest.java | 3 +- .../parser/microsoft/ooxml/xps/XPSParserTest.java | 9 +- .../parser/microsoft/pst/OutlookPSTParserTest.java | 3 +- .../tika/parser/microsoft/rtf/RTFParserTest.java | 5 +- .../parser/microsoft/xml/XML2003ParserTest.java | 5 +- .../tika-parser-miscoffice-module/pom.xml | 15 + .../java/org/apache/tika/parser/dbf/DBFParser.java | 5 +- .../apache/tika/parser/dif/DIFContentHandler.java | 9 +- .../org/apache/tika/parser/epub/EpubParser.java | 5 +- .../apache/tika/parser/hwp/HwpTextExtractorV5.java | 3 +- .../org/apache/tika/parser/hwp/HwpV5Parser.java | 3 +- .../indesign/ContentAndMetadataExtractor.java | 22 +- .../apache/tika/parser/indesign/IDMLParser.java | 12 +- .../apache/tika/parser/mif/MIFContentHandler.java | 7 +- .../java/org/apache/tika/parser/mif/MIFParser.java | 5 +- .../tika/parser/odf/FlatOpenDocumentParser.java | 3 +- .../tika/parser/odf/OpenDocumentMetaParser.java | 17 +- .../apache/tika/parser/odf/OpenDocumentParser.java | 5 +- .../tika/parser/wordperfect/QuattroProParser.java | 5 +- .../tika/parser/wordperfect/WordPerfectParser.java | 13 +- .../org/apache/tika/parser/dbf/DBFParserTest.java | 11 +- .../org/apache/tika/parser/dif/DIFParserTest.java | 6 +- .../apache/tika/parser/epub/EpubParserTest.java | 5 +- .../apache/tika/parser/hwp/HwpV5ParserTest.java | 5 +- .../tika/parser/ibooks/iBooksParserTest.java | 4 +- .../tika/parser/indesign/IDMLParserTest.java | 8 +- .../org/apache/tika/parser/mif/MIFParserTest.java | 3 +- .../org/apache/tika/parser/odf/ODFParserTest.java | 19 +- .../tika/parser/wordperfect/QuattroProTest.java | 8 +- .../tika/parser/wordperfect/WordPerfectTest.java | 14 +- .../apache/tika/parser/iptc/IptcAnpaParser.java | 3 +- .../apache/tika/parser/ocr/TesseractOCRConfig.java | 16 +- .../apache/tika/parser/ocr/TesseractOCRParser.java | 31 +- .../tika/parser/ocr/TesseractOCRConfigTest.java | 6 +- .../tika/parser/ocr/TesseractOCRParserTest.java | 2 +- .../resources/configs/TIKA-3582-tesseract.json | 2 +- .../configs/tika-config-tesseract-full.json | 2 +- .../configs/tika-config-tesseract-partial.json | 2 +- .../tika-parser-pdf-module/pom.xml | 15 + .../apache/tika/parser/pdf/AbstractPDF2XHTML.java | 44 +- .../java/org/apache/tika/parser/pdf/PDFParser.java | 28 +- .../tika/parser/pdf/PDMetadataExtractor.java | 53 +- .../org/apache/tika/parser/pdf/XFAExtractor.java | 21 +- .../tika/parser/pdf/image/ImageGraphicsEngine.java | 16 +- .../tika/renderer/pdf/pdfbox/PDFBoxRenderer.java | 12 +- .../tika/renderer/pdf/poppler/PopplerRenderer.java | 18 +- .../org/apache/tika/parser/pdf/PDFParserTest.java | 127 +- .../tika/parser/pkg/AbstractArchiveParser.java | 3 +- .../apache/tika/parser/pkg/CompressorParser.java | 3 +- .../org/apache/tika/parser/pkg/PackageParser.java | 9 +- .../java/org/apache/tika/parser/pkg/RarParser.java | 3 +- .../org/apache/tika/parser/pkg/SevenZParser.java | 3 +- .../org/apache/tika/parser/pkg/UnrarParser.java | 9 +- .../java/org/apache/tika/parser/pkg/ZipParser.java | 11 +- .../apache/tika/parser/pkg/AbstractPkgTest.java | 5 +- .../tika/parser/pkg/CompressorParserTest.java | 3 +- .../apache/tika/parser/csv/TextAndCSVParser.java | 19 +- .../tika/parser/markdown/MarkdownParser.java | 7 +- .../apache/tika/parser/strings/StringsConfig.java | 24 +- .../apache/tika/parser/strings/StringsParser.java | 22 +- .../java/org/apache/tika/parser/txt/TXTParser.java | 7 +- .../tika/parser/csv/TextAndCSVParserTest.java | 29 +- .../tika/parser/markdown/MarkdownParserTest.java | 9 +- .../tika/parser/strings/StringsConfigTest.java | 6 +- .../org/apache/tika/parser/txt/TXTParserTest.java | 49 +- .../configs/tika-config-strings-full.json | 2 +- .../configs/tika-config-strings-partial.json | 2 +- .../tika-parser-webarchive-module/pom.xml | 15 + .../org/apache/tika/parser/wacz/WACZParser.java | 3 +- .../org/apache/tika/parser/warc/WARCParser.java | 21 +- .../apache/tika/parser/http/HttpParserTest.java | 3 +- .../apache/tika/parser/wacz/WACZParserTest.java | 5 +- .../apache/tika/parser/warc/WARCParserTest.java | 11 +- .../tika-parser-xml-module/pom.xml | 15 + .../apache/tika/parser/tmx/TMXContentHandler.java | 26 +- .../java/org/apache/tika/parser/tmx/TMXParser.java | 3 +- .../tika/parser/xliff/XLIFF12ContentHandler.java | 14 +- .../apache/tika/parser/xliff/XLIFF12Parser.java | 3 +- .../tika/parser/xml/AbstractMetadataHandler.java | 33 +- .../xml/AttributeDependantMetadataHandler.java | 25 +- .../apache/tika/parser/xml/FictionBookParser.java | 3 +- .../apache/tika/parser/xml/MetadataHandler.java | 16 +- .../java/org/apache/tika/parser/xml/XMLParser.java | 5 +- .../apache/tika/parser/xml/DcXMLParserTest.java | 3 +- .../parser/xml/TextAndAttributeXMLParserTest.java | 3 +- .../org/apache/tika/parser/xmp/XmpExtractor.java | 6 +- .../org/apache/tika/async/cli/PluginsWriter.java | 10 +- .../apache/tika/async/cli/SimpleAsyncConfig.java | 16 +- .../org/apache/tika/async/cli/TikaAsyncCLI.java | 9 +- .../apache/tika/async/cli/AsyncCliParserTest.java | 12 +- .../apache/tika/async/cli/AsyncProcessorTest.java | 118 ++ .../tika/async/cli/TikaConfigAsyncWriterTest.java | 35 + .../configs/config-content-only-default.json | 2 - .../configs/config-content-only-dynamic.json | 2 - .../test/resources/configs/config-template.json | 2 - .../org/apache/tika/client/HttpClientFactory.java | 42 +- .../org/apache/tika/pipes/api/PipesResult.java | 13 +- .../tika/pipes/core/PerClientServerManager.java | 105 +- .../org/apache/tika/pipes/core/PipesClient.java | 89 +- .../org/apache/tika/pipes/core/PipesConfig.java | 153 ++- .../org/apache/tika/pipes/core/ServerManager.java | 4 +- .../tika/pipes/core/SharedServerManager.java | 10 +- .../apache/tika/pipes/core/async/AsyncEmitter.java | 7 + .../tika/pipes/core/async/AsyncProcessor.java | 20 +- .../core/extractor/FrictionlessUnpackHandler.java | 3 +- .../core/extractor/StandardUnpackSelector.java | 3 +- .../tika/pipes/core/extractor/UnpackExtractor.java | 3 +- .../tika/pipes/core/protocol/PipesMessage.java | 29 +- .../serialization/FetchEmitTupleDeserializer.java | 17 + .../core/serialization/JsonFetchEmitTupleList.java | 24 +- .../tika/pipes/core/server/ConnectionHandler.java | 87 +- .../apache/tika/pipes/core/server/EmitHandler.java | 43 +- .../tika/pipes/core/server/FetchHandler.java | 3 +- .../tika/pipes/core/server/ParseHandler.java | 11 +- .../apache/tika/pipes/core/server/PipesServer.java | 156 ++- .../apache/tika/pipes/core/server/PipesWorker.java | 2 +- .../tika/pipes/core/server/ServerProtocolIO.java | 29 + .../pipes/core/server/SharedServerResources.java | 2 +- .../tika/pipes/core/TikaPipesConfigTest.java | 8 + .../core/extractor/StandardUnpackSelectorTest.java | 3 +- .../tika/pipes/core/protocol/PipesMessageTest.java | 4 +- .../serialization/JsonFetchEmitTupleListTest.java | 28 + .../UserMetadataReservedKeyStanceTest.java | 60 + .../WireRestrictedFetchEmitTupleTest.java | 2 +- .../core/server/EmitHandlerUserMetadataTest.java | 76 ++ .../pipes/core/server/ServerProtocolIOTest.java | 72 + .../tika/pipes/fork/PipesForkParserTest.java | 5 +- .../pipes/core/DigestingOpenContainersTest.java | 3 +- .../apache/tika/pipes/core/EmbeddedLimitsTest.java | 6 +- .../pipes/core/PartialTimeoutPassbackTest.java | 80 ++ .../apache/tika/pipes/core/PassbackFilterTest.java | 9 +- .../apache/tika/pipes/core/PipesClientTest.java | 197 ++- .../pipes/core/async/AsyncChaosMonkeyTest.java | 3 +- .../configs/tika-config-timeout-lt-heartbeat.json | 4 +- .../pipes/pipesiterator/PipesIteratorBase.java | 14 +- .../tika/pipes/emitter/azblob/AZBlobEmitter.java | 5 +- .../tika-pipes-plugins/tika-pipes-csv/pom.xml | 1 + .../tika/pipes/iterator/csv/CSVPipesIterator.java | 14 +- .../pipes/iterator/csv/TestCSVPipesIterator.java | 20 + .../src/test/resources/test-reserved-header.csv | 2 + .../tika/pipes/reporter/es/ESReporterConfig.java | 12 +- .../apache/tika/pipes/es/ConfigExamplesTest.java | 18 + .../tika/pipes/fetcher/http/HttpFetcher.java | 22 +- .../fetcher/http/config/HttpFetcherConfig.java | 14 + .../tika/pipes/fetcher/http/HttpFetcherTest.java | 24 + .../pipes/iterator/jdbc/JDBCPipesIterator.java | 13 +- .../pipes/iterator/jdbc/TestJDBCPipesIterator.java | 25 +- .../apache/tika/pipes/fetcher/s3/S3Fetcher.java | 3 +- .../tika/config/loader/ComponentInstantiator.java | 5 + .../tika/serialization/ComponentNameResolver.java | 2 +- .../tika/serialization/ParseContextUtils.java | 20 +- .../java/org/apache/tika/config/AllLimitsTest.java | 2 +- .../org/apache/tika/config/TimeoutLimitsTest.java | 30 +- .../apache/tika/config/loader/TikaLoaderTest.java | 17 +- .../tika/metadata/filter/TestMetadataFilter.java | 69 +- .../StandardMetadataLimiterTest.java | 96 +- .../WireRestrictedParseContextTest.java | 14 + .../TIKA-4816-legacy-key-migration-default.json | 5 + .../configs/TIKA-4816-legacy-key-migration.json | 5 + tika-server/README.md | 20 +- .../sample-configs/vision/vlm-openai.json | 2 +- tika-server/pom.xml | 1 - tika-server/tika-server-client/pom.xml | 70 - .../tika/server/client/TikaAsyncHttpClient.java | 37 - .../org/apache/tika/server/client/TikaClient.java | 69 - .../apache/tika/server/client/TikaClientCLI.java | 153 --- .../server/client/TikaClientConfigException.java | 30 - .../tika/server/client/TikaEmitterResult.java | 54 - .../tika/server/client/TikaPipesHttpClient.java | 168 --- .../tika/server/client/TikaServerClientConfig.java | 106 -- .../org/apache/tika/server/client/TestBasic.java | 55 - .../src/test/resources/log4j2.xml | 32 - .../resources/tika-config-simple-fs-emitter.json | 11 - tika-server/tika-server-core/pom.xml | 43 - .../server/core/BadRequestExceptionMapper.java | 48 + .../tika/server/core/MaxRequestSizeFilter.java | 138 ++ .../org/apache/tika/server/core/ServerStatus.java | 2 +- .../org/apache/tika/server/core/TikaServerCli.java | 7 +- .../apache/tika/server/core/TikaServerConfig.java | 55 +- .../core/TikaServerParseExceptionMapper.java | 6 +- .../apache/tika/server/core/TikaServerProcess.java | 167 ++- .../tika/server/core/resource/AsyncResource.java | 85 +- .../server/core/resource/DetectorResource.java | 30 +- .../server/core/resource/LanguageResource.java | 56 +- .../server/core/resource/MetadataResource.java | 38 +- .../server/core/resource/PipesParsingHelper.java | 313 +++-- .../tika/server/core/resource/PipesResource.java | 106 +- .../core/resource/RecursiveMetadataResource.java | 8 +- .../tika/server/core/resource/TikaResource.java | 401 ++---- .../server/core/resource/TikaResourceAware.java | 18 +- .../tika/server/core/resource/TikaWelcome.java | 20 +- .../server/core/resource/TranslateResource.java | 137 -- .../server/core/resource/UnpackerResource.java | 2 - .../apache/tika/server/core/AsyncResourceTest.java | 242 ++++ .../org/apache/tika/server/core/CXFTestBase.java | 10 +- .../core/ConfigEndpointSecurityEnabledTest.java | 20 +- .../server/core/ConfigEndpointSecurityTest.java | 2 +- .../tika/server/core/LanguageResourceTest.java | 89 +- .../tika/server/core/MaxRequestSizeFilterTest.java | 127 ++ .../server/core/RecursiveMetadataResourceTest.java | 3 +- .../apache/tika/server/core/StackTraceTest.java | 87 +- .../org/apache/tika/server/core/TikaPipesTest.java | 112 +- .../apache/tika/server/core/TikaResourceTest.java | 155 ++- .../tika/server/core/TikaServerConfigTest.java | 9 + .../server/core/TikaServerIntegrationTest.java | 19 + .../core/TikaServerPipesIntegrationTest.java | 48 +- .../tika/server/core/TikaServerProcessTest.java | 36 + .../apache/tika/server/core/TikaWelcomeTest.java | 2 +- .../tika/server/core/TranslateResourceTest.java | 88 -- .../core/resource/PipesResponseBuilderTest.java | 90 ++ .../core/resource/ReservedComponentIdTest.java | 79 ++ .../test-documents/mock/hello_world_heading.xml} | 23 +- .../standard/resource/XMPMetadataResource.java | 21 +- .../tika/server/standard/DetectorResourceTest.java | 10 +- .../server/standard/JsonMaxFieldLengthTest.java | 2 +- .../tika/server/standard/MetadataResourceTest.java | 60 +- .../standard/RecursiveMetadataFilterTest.java | 5 +- .../standard/RecursiveMetadataResourceTest.java | 25 +- .../apache/tika/server/standard/TikaPipesTest.java | 4 +- .../tika/server/standard/TikaResourceTest.java | 48 +- .../UnpackerResourceConfigDisabledTest.java | 2 +- .../tika/server/standard/UnpackerResourceTest.java | 2 +- .../standard/UnpackerResourceWithConfigTest.java | 2 +- .../resource/ResourceServiceLoadingTest.java | 103 ++ .../main/java/org/apache/tika/xmp/XMPMetadata.java | 6 +- .../org/apache/tika/xmp/convert/TikaToXMP.java | 3 +- .../java/org/apache/tika/xmp/TikaToXMPTest.java | 5 +- 655 files changed, 16180 insertions(+), 6567 deletions(-) diff --cc CHANGES.txt index e46059d1a1,d2f1ae5bcf..6563f45e4e --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -8,6 -111,6 +111,9 @@@ Release 4.0.0 - ?? OTHER CHANGES ++ * OneNote extraction now follows document order, omits superseded page ++ revisions, and extracts embedded object BLOBs (TIKA-4814). ++ * MagicDetector now compiles its regular expression once, in the constructor, instead of recompiling it on every match (TIKA-4796). diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java index 8312bdb614,6c1355cb15..8b5a7eabbd --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackage.java @@@ -36,19 -31,13 +35,19 @@@ import java.util.regex.Matcher import java.util.regex.Pattern; import org.xml.sax.SAXException; +import org.xml.sax.helpers.AttributesImpl; import org.apache.tika.exception.TikaException; +import org.apache.tika.extractor.EmbeddedDocumentExtractor; +import org.apache.tika.extractor.EmbeddedDocumentUtil; +import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; - import org.apache.tika.metadata.Property; + import org.apache.tika.metadata.OneNote; import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.parser.ParseContext; import org.apache.tika.parser.microsoft.onenote.OneNotePropertyEnum; import org.apache.tika.parser.microsoft.onenote.OneNoteTreeWalkerOptions; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.ArrayNumber; import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.EightBytesOfData; import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.FourBytesOfData; import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.IProperty; @@@ -91,11 -76,6 +90,7 @@@ public class MSOneStorePackage private static final Pattern HYPERLINK_PATTERN = Pattern.compile("\uFDDFHYPERLINK\\s+\"([^\"]+)\"([^\"]+)$"); private static final String P = "p"; - /** - * A visible separator emitted between pages so they can be told apart in plain text - * output. - */ - private static final String PAGE_SEPARATOR = "----------------------------------------"; ++ private static final int MAX_TRAVERSAL_DEPTH = 100; static { LocalDateTime time32Epoch1980 = LocalDateTime.of(1980, Month.JANUARY, 1, 0, 0); @@@ -197,48 -162,93 +193,41 @@@ public void walkTree(OneNoteTreeWalkerOptions options, Metadata metadata, XHTMLContentHandler xhtml) throws SAXException, TikaException, IOException { - for (RevisionStoreObjectGroup revisionStoreObjectGroup : OtherFileNodeList) { - for (RevisionStoreObject revisionStoreObject : revisionStoreObjectGroup.objects) { - PropertySet propertySet = - revisionStoreObject.propertySet.objectSpaceObjectPropSet.body; - for (int i = 0; i < propertySet.rgData.size(); ++i) { - IProperty property = propertySet.rgData.get(i); - PropertyID propertyID = propertySet.rgPrids[i]; - PropertyType propertyType = PropertyType.fromIntVal(propertyID.type); - OneNotePropertyEnum oneNotePropertyEnum = - OneNotePropertyEnum.of(Unsigned.uint(propertyID.value).longValue()); - if (oneNotePropertyEnum == OneNotePropertyEnum.LastModifiedTimeStamp) { - long fullval = getScalar(property); - Instant instant = Instant.ofEpochSecond( - fullval / 10000000 + DATETIME_EPOCH_DIFF_1601); - if (instant.isAfter(lastModifiedTimestamp)) { - lastModifiedTimestamp = instant; - } - metadata.set(OneNote.LAST_MODIFIED_TIMESTAMP, - String.valueOf(lastModifiedTimestamp.toEpochMilli())); - } else if (oneNotePropertyEnum == OneNotePropertyEnum.CreationTimeStamp) { - // add the TIME32_EPOCH_DIFF_1980 because OneNote TIME32 epoch time is per 1980, not - // 1970 - long scalar = getScalar(property); - long creationTs = scalar + TIME32_EPOCH_DIFF_1980; - if (creationTs < creationTimestamp) { - creationTimestamp = creationTs; - } - metadata.set(OneNote.CREATION_TIMESTAMP, String.valueOf(creationTimestamp)); - } else if (oneNotePropertyEnum == OneNotePropertyEnum.LastModifiedTime) { - // add the TIME32_EPOCH_DIFF_1980 because OneNote TIME32 epoch time is per 1980, not - // 1970 - long scalar = getScalar(property); - long lastMod = scalar + TIME32_EPOCH_DIFF_1980; - if (lastMod > lastModified) { - lastModified = lastMod; - } - metadata.set(TikaCoreProperties.MODIFIED, String.valueOf(lastModified)); - } else if (oneNotePropertyEnum == OneNotePropertyEnum.Author) { - String author = - new String(((PrtFourBytesOfLengthFollowedByData) property).data, - StandardCharsets.UTF_8); - if (mostRecentAuthorProp) { - mostRecentAuthors.add(author); - } else if (originalAuthorProp) { - originalAuthors.add(author); - } else { - authors.add(author); - } - } else if (oneNotePropertyEnum == OneNotePropertyEnum.AuthorMostRecent) { - mostRecentAuthorProp = true; - } else if (oneNotePropertyEnum == OneNotePropertyEnum.AuthorOriginal) { - originalAuthorProp = true; - } else if (propertyType == PropertyType.FourBytesOfLengthFollowedByData) { - boolean isBinary = propertyIsBinary(oneNotePropertyEnum); - PrtFourBytesOfLengthFollowedByData dataProperty = - (PrtFourBytesOfLengthFollowedByData) property; - if ((dataProperty.data.length & 1) == 0 && - oneNotePropertyEnum != OneNotePropertyEnum.TextExtendedAscii && - !isBinary) { - if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) { - xhtml.startElement(P); - xhtml.characters( - new String(dataProperty.data, StandardCharsets.UTF_16LE)); - xhtml.endElement(P); - } - } else if (oneNotePropertyEnum == OneNotePropertyEnum.TextExtendedAscii) { - xhtml.startElement(P); - xhtml.characters( - new String(dataProperty.data, StandardCharsets.US_ASCII)); - xhtml.endElement(P); - } else if (!isBinary) { - if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) { - xhtml.startElement(P); - xhtml.characters( - new String(dataProperty.data, StandardCharsets.UTF_16LE)); - xhtml.endElement(P); - } - } else { - if (oneNotePropertyEnum == OneNotePropertyEnum.RichEditTextUnicode) { - handleRichEditTextUnicode(dataProperty.data, xhtml); - } else { - //TODO -- these seem to be somewhat broken font files and other - //odds and ends...what are they and how should we process them? - //handleEmbedded(content.size()); - } - } - } + walkTree(options, metadata, xhtml, new ParseContext()); + } + + public void walkTree(OneNoteTreeWalkerOptions options, Metadata metadata, + XHTMLContentHandler xhtml, ParseContext parseContext) + throws SAXException, TikaException, IOException { + this.parseContext = parseContext; + this.parentMetadata = metadata; + this.embeddedDocumentExtractor = + EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(parseContext); + if (!cells.isEmpty()) { + // Walk each page cell (object space) as a tree, starting from the root objects of + // its current revision and following the object references in property order. This + // emits the text in document order. The pages are walked in the order in which the + // section object space references them; cells that hold older versions of a page + // (the same object space in a different revision context) are skipped. + List<RevisionStoreCell> pageCells = new ArrayList<>(); + List<RevisionStoreCell> otherCells = new ArrayList<>(); + splitCells(pageCells, otherCells); - boolean firstPage = true; + for (RevisionStoreCell cell : pageCells) { - if (!firstPage) { - xhtml.startElement(P); - xhtml.characters(PAGE_SEPARATOR); - xhtml.endElement(P); - } - firstPage = false; + xhtml.startElement("div", "class", "page"); + walkCell(cell, options, metadata, xhtml); + xhtml.endElement("div"); + } + for (RevisionStoreCell cell : otherCells) { + walkCell(cell, options, metadata, xhtml); + } + } else { + // no cell information available - walk the object groups in revision order + Map<ExGuid, RevisionStoreObject> objectsById = indexObjectsById(OtherFileNodeList); + Set<ExGuid> visited = new HashSet<>(); + for (RevisionStoreObjectGroup objectGroup : OtherFileNodeList) { + for (RevisionStoreObject object : objectGroup.objects) { + walkObject(object, objectsById, visited, AuthorRole.NONE, options, metadata, - xhtml); ++ xhtml, 0); } } } @@@ -255,415 -265,6 +244,435 @@@ } } + /** + * Splits the cells into page cells, ordered as the section object space references them, + * and the remaining cells. A cell that holds an older version of a page - the same object + * space referenced by a page cell, but in a different revision context - is dropped, so + * content is not emitted once per version snapshot. + */ + private void splitCells(List<RevisionStoreCell> pageCells, - List<RevisionStoreCell> otherCells) { ++ List<RevisionStoreCell> otherCells) throws TikaException { + List<CellID> orderedCellIds = collectSectionReferencedCells(); + if (orderedCellIds.isEmpty()) { + // no page ordering information available - process the cells in storage order + pageCells.addAll(cells); + return; + } + Map<CellID, RevisionStoreCell> remainingCells = new LinkedHashMap<>(); + for (RevisionStoreCell cell : cells) { + remainingCells.put(cell.cellID, cell); + } + Set<ExGuid> coveredObjectSpaces = new HashSet<>(); + for (CellID cellId : orderedCellIds) { + RevisionStoreCell cell = remainingCells.remove(cellId); + if (cell != null) { + pageCells.add(cell); + coveredObjectSpaces.add(cellId.extendGUID2); + } + } + for (RevisionStoreCell cell : remainingCells.values()) { + if (cell.cellID == null || + !coveredObjectSpaces.contains(cell.cellID.extendGUID2)) { + // not an older version of one of the pages - keep it so no content is lost + otherCells.add(cell); + } + } + } + + /** + * Walks the section object space (the data root cell) and collects the object space (cell) + * references in document order - this is the order of the pages in the section. + */ - private List<CellID> collectSectionReferencedCells() { ++ private List<CellID> collectSectionReferencedCells() throws TikaException { + List<CellID> orderedCellIds = new ArrayList<>(); + if (dataRootCell == null) { + return orderedCellIds; + } + Map<ExGuid, RevisionStoreObject> objectsById = indexObjectsById(dataRootCell.objectGroups); + Set<ExGuid> visited = new HashSet<>(); + for (RevisionManifestRootDeclare rootDeclare : dataRootCell.rootDeclares) { + collectReferencedCells(objectsById.get(rootDeclare.objectExGuid), objectsById, visited, - orderedCellIds); ++ orderedCellIds, 0); + } + for (RevisionStoreObjectGroup objectGroup : dataRootCell.objectGroups) { + for (RevisionStoreObject object : objectGroup.objects) { - collectReferencedCells(object, objectsById, visited, orderedCellIds); ++ collectReferencedCells(object, objectsById, visited, orderedCellIds, 0); + } + } + return orderedCellIds; + } + + private void collectReferencedCells(RevisionStoreObject object, + Map<ExGuid, RevisionStoreObject> objectsById, - Set<ExGuid> visited, List<CellID> out) { ++ Set<ExGuid> visited, List<CellID> out, int depth) ++ throws TikaException { ++ if (depth > MAX_TRAVERSAL_DEPTH) { ++ return; ++ } + if (object == null || object.propertySet == null || + object.propertySet.objectSpaceObjectPropSet == null) { + return; + } + if (object.objectID != null && !visited.add(object.objectID)) { + return; + } + List<PropertyAction> actions = collectObjectActions(object); + for (PropertyAction action : actions) { + if (action.spaceReference != null) { + out.add(action.spaceReference); + } else if (action.isChildReference && action.childReference != null) { + collectReferencedCells(objectsById.get(action.childReference), objectsById, - visited, out); ++ visited, out, depth + 1); + } + } + } + + private void walkCell(RevisionStoreCell cell, OneNoteTreeWalkerOptions options, + Metadata metadata, XHTMLContentHandler xhtml) + throws SAXException, TikaException, IOException { + Map<ExGuid, RevisionStoreObject> objectsById = indexObjectsById(cell.objectGroups); + Set<ExGuid> visited = new HashSet<>(); + // Only objects reachable from the root objects of the current revision are part of + // the current content. The object groups may also contain older, superseded versions + // of objects (under a different object ID); those are intentionally not walked. + for (RevisionManifestRootDeclare rootDeclare : cell.rootDeclares) { + walkObject(objectsById.get(rootDeclare.objectExGuid), objectsById, visited, - AuthorRole.NONE, options, metadata, xhtml); ++ AuthorRole.NONE, options, metadata, xhtml, 0); + } + if (visited.isEmpty()) { + // no root objects could be resolved - walk everything so no content is lost + for (RevisionStoreObjectGroup objectGroup : cell.objectGroups) { + for (RevisionStoreObject object : objectGroup.objects) { + walkObject(object, objectsById, visited, AuthorRole.NONE, options, metadata, - xhtml); ++ xhtml, 0); + } + } + } + } + + /** + * Builds a map of object ID to object. The object groups are ordered from the oldest + * revision to the newest, so a newer version of an object wins over an older one. + */ + private Map<ExGuid, RevisionStoreObject> indexObjectsById( + List<RevisionStoreObjectGroup> objectGroups) { + Map<ExGuid, RevisionStoreObject> objectsById = new HashMap<>(); + for (RevisionStoreObjectGroup objectGroup : objectGroups) { + for (RevisionStoreObject object : objectGroup.objects) { + if (object.objectID != null) { + objectsById.put(object.objectID, object); + } + } + } + return objectsById; + } + + private void walkObject(RevisionStoreObject object, + Map<ExGuid, RevisionStoreObject> objectsById, Set<ExGuid> visited, + AuthorRole authorRole, OneNoteTreeWalkerOptions options, - Metadata metadata, XHTMLContentHandler xhtml) ++ Metadata metadata, XHTMLContentHandler xhtml, int depth) + throws SAXException, TikaException, IOException { ++ if (depth > MAX_TRAVERSAL_DEPTH) { ++ throw new TikaException("OneNote object graph exceeds maximum depth of " + ++ MAX_TRAVERSAL_DEPTH); ++ } + if (object == null) { + return; + } + if (object.objectID != null && !visited.add(object.objectID)) { + return; + } + if (object.fileDataObject != null) { + // the object carries opaque binary data, e.g. an embedded image or file + handleEmbedded(object.fileDataObject.getData(), xhtml); + } + if (object.propertySet == null || + object.propertySet.objectSpaceObjectPropSet == null) { + return; + } + List<PropertyAction> actions = collectObjectActions(object); + // An image node can reference the same picture twice: PictureContainer holds the + // canonical image data and WebPictureContainer14 holds a rendition derived from it + // (e.g. re-rendered when the picture was resized). Only emit the derived rendition + // when the canonical container is missing, so the picture is not extracted twice. + boolean hasPrimaryPicture = false; + for (PropertyAction action : actions) { + if (action.isChildReference && action.childReference != null && + action.oneNotePropertyEnum == OneNotePropertyEnum.PictureContainer) { + hasPrimaryPicture = true; + break; + } + } + // The title structure of a page (StructureElementChildNodes) appears above the page + // body on screen, but is declared after the body child nodes. Emit it first so the + // text comes out in visual order. + for (PropertyAction action : actions) { + if (action.oneNotePropertyEnum == OneNotePropertyEnum.StructureElementChildNodes) { - processAction(action, objectsById, visited, authorRole, options, metadata, xhtml); ++ processAction(action, objectsById, visited, authorRole, options, metadata, xhtml, ++ depth); + } + } + for (PropertyAction action : actions) { + if (hasPrimaryPicture && + action.oneNotePropertyEnum == OneNotePropertyEnum.WebPictureContainer14) { + continue; + } + if (action.oneNotePropertyEnum != OneNotePropertyEnum.StructureElementChildNodes) { - processAction(action, objectsById, visited, authorRole, options, metadata, xhtml); ++ processAction(action, objectsById, visited, authorRole, options, metadata, xhtml, ++ depth); + } + } + } + + /** + * A property of an object, together with the object reference assigned to it if it is + * an object reference property. + */ + private static final class PropertyAction { + private final IProperty property; + private final PropertyType propertyType; + private final OneNotePropertyEnum oneNotePropertyEnum; + private final boolean isChildReference; + private final ExGuid childReference; + private final CellID spaceReference; + + PropertyAction(IProperty property, PropertyType propertyType, + OneNotePropertyEnum oneNotePropertyEnum, boolean isChildReference, + ExGuid childReference, CellID spaceReference) { + this.property = property; + this.propertyType = propertyType; + this.oneNotePropertyEnum = oneNotePropertyEnum; + this.isChildReference = isChildReference; + this.childReference = childReference; + this.spaceReference = spaceReference; + } + } + + /** + * Flattens the properties of an object, in order, into a list of actions. + */ - private List<PropertyAction> collectObjectActions(RevisionStoreObject object) { ++ private List<PropertyAction> collectObjectActions(RevisionStoreObject object) ++ throws TikaException { + List<ExGuid> referencedObjects = + object.referencedObjectID == null || object.referencedObjectID.content == null ? + Collections.emptyList() : object.referencedObjectID.content; + List<CellID> referencedSpaces = object.referencedObjectSpacesID == null || + object.referencedObjectSpacesID.content == null ? Collections.emptyList() : + object.referencedObjectSpacesID.content; + List<PropertyAction> actions = new ArrayList<>(); + collectActions(object.propertySet.objectSpaceObjectPropSet.body, referencedObjects, - new int[]{0}, referencedSpaces, new int[]{0}, actions); ++ new int[]{0}, referencedSpaces, new int[]{0}, actions, 0); + return actions; + } + + /** + * Flattens the properties of a property set, in order, into a list of actions. + * Properties of type ObjectID or ArrayOfObjectIDs consume, in property order, the object + * references of the containing object, and properties of type ObjectSpaceID or + * ArrayOfObjectSpaceIDs consume the object space (cell) references (see MS-ONESTORE + * section 2.7.8), so the references must be assigned here, in property order, no matter + * in which order the actions are processed later. + */ + private void collectActions(PropertySet propertySet, List<ExGuid> referencedObjects, + int[] referenceCursor, List<CellID> referencedSpaces, - int[] spaceCursor, List<PropertyAction> actions) { ++ int[] spaceCursor, List<PropertyAction> actions, int depth) ++ throws TikaException { ++ if (depth > MAX_TRAVERSAL_DEPTH) { ++ throw new TikaException("OneNote property graph exceeds maximum depth of " + ++ MAX_TRAVERSAL_DEPTH); ++ } + if (propertySet == null || propertySet.rgPrids == null || propertySet.rgData == null) { + return; + } + for (int i = 0; i < propertySet.rgPrids.length && i < propertySet.rgData.size(); ++i) { + IProperty property = propertySet.rgData.get(i); + PropertyID propertyID = propertySet.rgPrids[i]; + PropertyType propertyType = PropertyType.fromIntVal(propertyID.type); + OneNotePropertyEnum oneNotePropertyEnum = + OneNotePropertyEnum.of(Unsigned.uint(propertyID.value).longValue()); + if (propertyType == PropertyType.ObjectID) { + actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum, true, + nextReference(referencedObjects, referenceCursor), null)); + } else if (propertyType == PropertyType.ArrayOfObjectIDs) { - int count = property instanceof ArrayNumber ? ((ArrayNumber) property).number : 0; ++ int count = property instanceof ArrayNumber ? ++ Math.min(Math.max(0, ((ArrayNumber) property).number), ++ referencedObjects.size() - referenceCursor[0]) : 0; + for (int j = 0; j < count; ++j) { + actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum, + true, nextReference(referencedObjects, referenceCursor), null)); + } + } else if (propertyType == PropertyType.ObjectSpaceID) { + actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum, false, + null, nextSpaceReference(referencedSpaces, spaceCursor))); + } else if (propertyType == PropertyType.ArrayOfObjectSpaceIDs) { - int count = property instanceof ArrayNumber ? ((ArrayNumber) property).number : 0; ++ int count = property instanceof ArrayNumber ? ++ Math.min(Math.max(0, ((ArrayNumber) property).number), ++ referencedSpaces.size() - spaceCursor[0]) : 0; + for (int j = 0; j < count; ++j) { + actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum, + false, null, nextSpaceReference(referencedSpaces, spaceCursor))); + } + } else if (propertyType == PropertyType.PropertySet) { + if (property instanceof PropertySet) { + collectActions((PropertySet) property, referencedObjects, referenceCursor, - referencedSpaces, spaceCursor, actions); ++ referencedSpaces, spaceCursor, actions, depth + 1); + } + } else if (propertyType == PropertyType.ArrayOfPropertyValues) { + if (property instanceof PrtArrayOfPropertyValues && + ((PrtArrayOfPropertyValues) property).data != null) { + for (PropertySet nested : ((PrtArrayOfPropertyValues) property).data) { + collectActions(nested, referencedObjects, referenceCursor, - referencedSpaces, spaceCursor, actions); ++ referencedSpaces, spaceCursor, actions, depth + 1); + } + } + } else { + actions.add(new PropertyAction(property, propertyType, oneNotePropertyEnum, false, + null, null)); + } + } + } + + private void processAction(PropertyAction action, + Map<ExGuid, RevisionStoreObject> objectsById, Set<ExGuid> visited, + AuthorRole authorRole, OneNoteTreeWalkerOptions options, - Metadata metadata, XHTMLContentHandler xhtml) ++ Metadata metadata, XHTMLContentHandler xhtml, int depth) + throws SAXException, TikaException, IOException { + if (action.spaceReference != null) { + // a reference to another object space (cell) - cells are walked separately + return; + } + if (action.isChildReference) { + AuthorRole childRole = AuthorRole.NONE; + if (action.oneNotePropertyEnum == OneNotePropertyEnum.AuthorMostRecent) { + childRole = AuthorRole.MOST_RECENT; + } else if (action.oneNotePropertyEnum == OneNotePropertyEnum.AuthorOriginal) { + childRole = AuthorRole.ORIGINAL; + } + walkObject(action.childReference == null ? null : + objectsById.get(action.childReference), objectsById, visited, - childRole, options, metadata, xhtml); ++ childRole, options, metadata, xhtml, depth + 1); + } else { + processPrimitiveProperty(action.property, action.propertyType, + action.oneNotePropertyEnum, authorRole, options, metadata, xhtml); + } + } + + private ExGuid nextReference(List<ExGuid> referencedObjects, int[] referenceCursor) { + if (referenceCursor[0] < referencedObjects.size()) { + return referencedObjects.get(referenceCursor[0]++); + } + return null; + } + + private CellID nextSpaceReference(List<CellID> referencedSpaces, int[] spaceCursor) { + if (spaceCursor[0] < referencedSpaces.size()) { + return referencedSpaces.get(spaceCursor[0]++); + } + return null; + } + + private void processPrimitiveProperty(IProperty property, PropertyType propertyType, + OneNotePropertyEnum oneNotePropertyEnum, + AuthorRole authorRole, + OneNoteTreeWalkerOptions options, Metadata metadata, + XHTMLContentHandler xhtml) + throws SAXException, TikaException, IOException { + if (oneNotePropertyEnum == OneNotePropertyEnum.LastModifiedTimeStamp) { + long fullval = getScalar(property); + Instant instant = Instant.ofEpochSecond( + fullval / 10000000 + DATETIME_EPOCH_DIFF_1601); + if (instant.isAfter(lastModifiedTimestamp)) { + lastModifiedTimestamp = instant; + } - metadata.set(ONE_NOTE_PREFIX + "lastModifiedTimestamp", ++ metadata.set(OneNote.LAST_MODIFIED_TIMESTAMP, + String.valueOf(lastModifiedTimestamp.toEpochMilli())); + } else if (oneNotePropertyEnum == OneNotePropertyEnum.CreationTimeStamp) { + // add the TIME32_EPOCH_DIFF_1980 because OneNote TIME32 epoch time is per 1980, not + // 1970 + long scalar = getScalar(property); + long creationTs = scalar + TIME32_EPOCH_DIFF_1980; + if (creationTs < creationTimestamp) { + creationTimestamp = creationTs; + } - metadata.set(ONE_NOTE_PREFIX + "creationTimestamp", String.valueOf(creationTimestamp)); ++ metadata.set(OneNote.CREATION_TIMESTAMP, String.valueOf(creationTimestamp)); + } else if (oneNotePropertyEnum == OneNotePropertyEnum.LastModifiedTime) { + // add the TIME32_EPOCH_DIFF_1980 because OneNote TIME32 epoch time is per 1980, not + // 1970 + long scalar = getScalar(property); + long lastMod = scalar + TIME32_EPOCH_DIFF_1980; + if (lastMod > lastModified) { + lastModified = lastMod; + } + metadata.set(TikaCoreProperties.MODIFIED, String.valueOf(lastModified)); + } else if (oneNotePropertyEnum == OneNotePropertyEnum.Author) { + String author = new String(((PrtFourBytesOfLengthFollowedByData) property).data, + StandardCharsets.UTF_8); + if (authorRole == AuthorRole.MOST_RECENT) { + mostRecentAuthors.add(author); + } else if (authorRole == AuthorRole.ORIGINAL) { + originalAuthors.add(author); + // the original authors are the creators of the content + authors.add(author); + } else { + authors.add(author); + } + } else if (propertyType == PropertyType.FourBytesOfLengthFollowedByData) { + boolean isBinary = propertyIsBinary(oneNotePropertyEnum); + PrtFourBytesOfLengthFollowedByData dataProperty = + (PrtFourBytesOfLengthFollowedByData) property; + if ((dataProperty.data.length & 1) == 0 && + oneNotePropertyEnum != OneNotePropertyEnum.TextExtendedAscii && !isBinary) { + if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) { + xhtml.startElement(P); + xhtml.characters(new String(dataProperty.data, StandardCharsets.UTF_16LE)); + xhtml.endElement(P); + } + } else if (oneNotePropertyEnum == OneNotePropertyEnum.TextExtendedAscii) { + xhtml.startElement(P); + xhtml.characters(new String(dataProperty.data, StandardCharsets.US_ASCII)); + xhtml.endElement(P); + } else if (!isBinary) { + if (options.getUtf16PropertiesToPrint().contains(oneNotePropertyEnum)) { + xhtml.startElement(P); + xhtml.characters(new String(dataProperty.data, StandardCharsets.UTF_16LE)); + xhtml.endElement(P); + } + } else { + if (oneNotePropertyEnum == OneNotePropertyEnum.RichEditTextUnicode) { + handleRichEditTextUnicode(dataProperty.data, xhtml); + } else { + //TODO -- these seem to be somewhat broken font files and other + //odds and ends...what are they and how should we process them? + //handleEmbedded(content.size()); + } + } + } + } + + + /** + * Hands the binary data of an embedded object (e.g. an image or an attached file) to the + * embedded document extractor. + */ + private void handleEmbedded(byte[] data, XHTMLContentHandler xhtml) + throws SAXException, IOException { + if (data == null || data.length == 0 || embeddedDocumentExtractor == null) { + return; + } + Metadata embeddedMetadata = Metadata.newInstance(this.parseContext); + AttributesImpl attributes = new AttributesImpl(); + attributes.addAttribute("", "class", "class", "CDATA", "embedded"); + xhtml.startElement("div", attributes); + xhtml.endElement("div"); + try (TikaInputStream tis = TikaInputStream.get(data)) { + if (embeddedDocumentExtractor.shouldParseEmbedded(embeddedMetadata)) { + embeddedDocumentExtractor.parseEmbedded(tis, new EmbeddedContentHandler(xhtml), + embeddedMetadata, this.parseContext, false); + } + } catch (IOException e) { + EmbeddedDocumentUtil.recordEmbeddedStreamException(e, parentMetadata); + } + } private void handleRichEditTextUnicode(byte[] arr, XHTMLContentHandler xhtml) throws SAXException, IOException, TikaException { diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParser.java index a44a9f75ed,25790ead8b..30ce905c67 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParser.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStoreParser.java @@@ -69,10 -55,8 +69,11 @@@ public class MSOneStoreParser private List<DataElement> objectGroupDataElements; // The DataElements of Object BLOB private List<DataElement> objectBlOBElements; + // The DataElements of Object BLOB, keyed by their data element extended GUID + private Map<ExGuid, DataElement> objectBlOBElementsById; public MSOneStorePackage parse(DataElementPackage dataElementPackage) throws IOException { ++ storageIndexHashTab.clear(); MSOneStorePackage msOneStorePackage = new MSOneStorePackage(); storageIndexDataElements = dataElementPackage.dataElements.stream() diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java index 6de0064661,afd2f6c067..dfa3f05d8f --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/streamobj/RevisionStoreObjectGroup.java @@@ -83,6 -73,6 +83,9 @@@ public class RevisionStoreObjectGroup for (int i = 0; i < dataObject.objectGroupDeclarations.objectGroupObjectBLOBDataDeclarationList.size(); i++) { ++ if (i >= dataObject.objectGroupData.objectGroupObjectDataBLOBReferenceList.size()) { ++ throw new IOException("Missing BLOB reference for object declaration " + i); ++ } ObjectGroupObjectBLOBDataDeclaration objectGroupObjectBLOBDataDeclaration = dataObject.objectGroupDeclarations.objectGroupObjectBLOBDataDeclarationList.get( i); diff --cc tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java index 841415bb98,0000000000..e164455dab mode 100644,000000..100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/onenote/fsshttpb/MSOneStorePackageTest.java @@@ -1,293 -1,0 +1,292 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.onenote.fsshttpb; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.microsoft.onenote.OneNoteTreeWalkerOptions; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.ArrayNumber; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.IProperty; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.NoData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.PrtArrayOfPropertyValues; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.property.PrtFourBytesOfLengthFollowedByData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.ObjectGroupObjectData; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.PropertySet; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.PropertySetObject; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionManifestRootDeclare; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreCell; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObject; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.RevisionStoreObjectGroup; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.CellID; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.ExGuid; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.PropertyID; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.PropertyType; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.space.ObjectSpaceObjectPropSet; +import org.apache.tika.parser.microsoft.onenote.fsshttpb.util.ByteUtil; +import org.apache.tika.sax.ToTextContentHandler; +import org.apache.tika.sax.XHTMLContentHandler; + +public class MSOneStorePackageTest { + + @Test + public void testPagesFollowSectionOrderAndDropOlderCellVersions() throws Exception { + ExGuid sectionRootId = id(1); + CellID pageOne = cell(10, 100); + CellID pageTwo = cell(20, 200); + CellID oldPageOne = cell(11, 100); + RevisionStoreCell pageTwoCell = cellWithText(pageTwo, "page two"); + RevisionStoreCell oldPageOneCell = cellWithText(oldPageOne, "old page one"); + RevisionStoreCell pageOneCell = cellWithText(pageOne, "page one"); + RevisionStoreCell unrelatedCell = cellWithText(cell(30, 300), "unrelated"); + + RevisionStoreObject sectionRoot = object(sectionRootId, + propertySet(new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D78, + new NoData()), + new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D79, + new NoData())), + Collections.emptyList(), Arrays.asList(pageOne, pageTwo)); + RevisionStoreCell section = new RevisionStoreCell(); + section.objectGroups.add(group(sectionRoot)); + RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare(); + rootDeclare.objectExGuid = sectionRootId; + section.rootDeclares.add(rootDeclare); + RevisionManifestRootDeclare missingRoot = new RevisionManifestRootDeclare(); + missingRoot.objectExGuid = id(1000); + section.rootDeclares.add(missingRoot); + + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.dataRootCell = section; + pkg.cells.addAll(Arrays.asList(pageTwoCell, oldPageOneCell, pageOneCell, unrelatedCell)); + + String text = walk(pkg); + assertTrue(text.indexOf("page one") < text.indexOf("page two")); - assertTrue(text.contains("----------------------------------------")); + assertFalse(text.contains("old page one")); + assertTrue(text.contains("unrelated")); + } + + @Test + public void testUnresolvedRootsFallBackToAllObjects() throws Exception { + RevisionStoreCell cell = cellWithText(cell(1, 1), "fallback content"); + cell.rootDeclares.clear(); + RevisionManifestRootDeclare missingRoot = new RevisionManifestRootDeclare(); + missingRoot.objectExGuid = id(999); + cell.rootDeclares.add(missingRoot); + + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.cells.add(cell); + + assertTrue(walk(pkg).contains("fallback content")); + } + + @Test + public void testPrimaryPictureSuppressesDerivedPicture() throws Exception { + ExGuid pictureID = id(40); + ExGuid webPictureID = id(41); + RevisionStoreObject root = object(id(42), propertySet( + new PropertySpec(PropertyType.ObjectID, 0x20001C3F, new NoData()), + new PropertySpec(PropertyType.ObjectID, 0x200034C8, new NoData())), + Arrays.asList(pictureID, webPictureID), Collections.emptyList()); + RevisionStoreObject picture = object(pictureID, propertySet(), + Collections.emptyList(), Collections.emptyList()); + picture.propertySet.objectSpaceObjectPropSet.body = null; + RevisionStoreObject webPicture = object(webPictureID, propertySet( + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text("derived picture"))), + Collections.emptyList(), Collections.emptyList()); + RevisionStoreCell cell = new RevisionStoreCell(); + cell.objectGroups.add(group(root, picture, webPicture)); + RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare(); + rootDeclare.objectExGuid = root.objectID; + cell.rootDeclares.add(rootDeclare); + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.cells.add(cell); + + assertFalse(walk(pkg).contains("derived picture")); + } + + @Test + public void testNestedPropertySetsAndMissingReferencesAreTraversedSafely() throws Exception { + ExGuid childId = id(2); + PropertySet nested = propertySet(new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text("nested text"))); + PrtArrayOfPropertyValues array = new PrtArrayOfPropertyValues(); + array.data = new PropertySet[]{propertySet(new PropertySpec( + PropertyType.FourBytesOfLengthFollowedByData, 0x1C003498, text("array text")))}; + RevisionStoreObject root = object(id(1), propertySet( + new PropertySpec(PropertyType.PropertySet, 0, nested), + new PropertySpec(PropertyType.ArrayOfPropertyValues, 0, array), + new PropertySpec(PropertyType.ArrayOfObjectIDs, 0x24001D5F, + arrayNumber(2)), + new PropertySpec(PropertyType.ObjectID, 0x24001D5F, new NoData()), + new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D78, new NoData()), + new PropertySpec(PropertyType.ObjectSpaceID, 0x20001D79, new NoData()), + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C001DD7, bytes((byte) 'u', (byte) 0, (byte) 1)), + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C001C22, bytes((byte) 'h', (byte) 0, (byte) 'i', (byte) 0, + (byte) 0, (byte) 0))), + Collections.singletonList(childId), Collections.singletonList(cell(50, 51))); + RevisionStoreObject child = object(childId, propertySet( + new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text("child text"))), + Collections.emptyList(), Collections.emptyList()); + RevisionStoreCell cell = new RevisionStoreCell(); + cell.objectGroups.add(group(root, child)); + RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare(); + rootDeclare.objectExGuid = root.objectID; + cell.rootDeclares.add(rootDeclare); + + MSOneStorePackage pkg = new MSOneStorePackage(); + pkg.cells.add(cell); + String text = walk(pkg); + assertTrue(text.contains("nested text")); + assertTrue(text.contains("array text")); + assertTrue(text.contains("child text")); + assertTrue(text.contains("u")); + assertTrue(text.contains("hi")); + } + + private static String walk(MSOneStorePackage pkg) throws Exception { + Metadata metadata = new Metadata(); + StringWriter writer = new StringWriter(); + XHTMLContentHandler xhtml = new XHTMLContentHandler( + new ToTextContentHandler(writer), metadata, new ParseContext()); + xhtml.startDocument(); + pkg.walkTree(new OneNoteTreeWalkerOptions(), metadata, xhtml); + xhtml.endDocument(); + return writer.toString(); + } + + private static RevisionStoreCell cellWithText(CellID cellID, String value) throws Exception { + RevisionStoreObject object = object(id(cellID.extendGUID1.hashCode()), + propertySet(new PropertySpec(PropertyType.FourBytesOfLengthFollowedByData, + 0x1C003498, text(value))), Collections.emptyList(), Collections.emptyList()); + RevisionStoreCell cell = new RevisionStoreCell(); + cell.cellID = cellID; + cell.objectGroups.add(group(object)); + RevisionManifestRootDeclare rootDeclare = new RevisionManifestRootDeclare(); + rootDeclare.objectExGuid = object.objectID; + cell.rootDeclares.add(rootDeclare); + return cell; + } + + private static RevisionStoreObject object(ExGuid objectID, PropertySet body, + List<ExGuid> references, List<CellID> spaces) + throws Exception { + RevisionStoreObject object = new RevisionStoreObject(); + object.objectID = objectID; + PropertySetObject propertySetObject = new PropertySetObject(null, emptyObjectData()); + ObjectSpaceObjectPropSet propSet = new ObjectSpaceObjectPropSet(); + propSet.body = body; + propertySetObject.objectSpaceObjectPropSet = propSet; + object.propertySet = propertySetObject; + if (!references.isEmpty()) { + object.referencedObjectID = new org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.ExGUIDArray(); + object.referencedObjectID.content = references; + } + if (!spaces.isEmpty()) { + object.referencedObjectSpacesID = new org.apache.tika.parser.microsoft.onenote.fsshttpb.streamobj.basic.CellIDArray(); + object.referencedObjectSpacesID.content = spaces; + } + return object; + } + + private static RevisionStoreObjectGroup group(RevisionStoreObject... objects) { + RevisionStoreObjectGroup group = new RevisionStoreObjectGroup(id(500)); + group.objects.addAll(Arrays.asList(objects)); + return group; + } + + private static PropertySet propertySet(PropertySpec... specs) { + PropertySet set = new PropertySet(); + set.cProperties = specs.length; + set.rgPrids = new PropertyID[specs.length]; + set.rgData = new ArrayList<>(); + for (int i = 0; i < specs.length; i++) { + set.rgPrids[i] = propertyID(specs[i].type, specs[i].value); + set.rgData.add(specs[i].property); + } + return set; + } + + private static PropertyID propertyID(PropertyType type, int value) { + PropertyID id = new PropertyID(); + id.type = type.getIntVal(); + id.value = value; + return id; + } + + private static PrtFourBytesOfLengthFollowedByData bytes(byte... value) { + PrtFourBytesOfLengthFollowedByData data = new PrtFourBytesOfLengthFollowedByData(); + data.data = value; + data.cb = data.data.length; + return data; + } + + private static PrtFourBytesOfLengthFollowedByData text(String value) { + PrtFourBytesOfLengthFollowedByData data = new PrtFourBytesOfLengthFollowedByData(); + data.data = value.getBytes(StandardCharsets.US_ASCII); + data.cb = data.data.length; + return data; + } + + private static ArrayNumber arrayNumber(int number) { + ArrayNumber array = new ArrayNumber(); + array.number = number; + return array; + } + + private static ObjectGroupObjectData emptyObjectData() { + ObjectGroupObjectData data = new ObjectGroupObjectData(); + data.data.content.addAll(ByteUtil.toListOfByte(new byte[]{0, 0, 0, (byte) 0x80, + 0, 0, 0, 0})); + return data; + } + + private static CellID cell(int first, int second) { + return new CellID(id(first), id(second)); + } + + private static ExGuid id(int value) { + return new ExGuid(value, UUID.nameUUIDFromBytes(("id-" + value).getBytes(StandardCharsets.UTF_8))); + } + + private static final class PropertySpec { + private final PropertyType type; + private final int value; + private final IProperty property; + + private PropertySpec(PropertyType type, int value, IProperty property) { + this.type = type; + this.value = value; + this.property = property; + } + } +}
