This is an automated email from the ASF dual-hosted git repository.
tballison pushed a change to branch TIKA-4816-metadata-key-api
in repository https://gitbox.apache.org/repos/asf/tika.git
from 0e3fe95059 TIKA-4816 fix EMBEDDED_EXCEPTION erasure under active
metadata limiter
new a5924eb8a7 TIKA-4816 metadata-key stage 5a: registry scope expansion +
module attribution + key-shape renames
new e60518e708 TIKA-4816 stages 5-6: category-A migration, forbiddenapis
bans, key hygiene
new 7672ca16fa TIKA-4816 stage 7: flip reserved-key guard from silent-drop
to throw
add 3d5bd05698 TIKA-4813: unified timeout model (#3005)
add 787d9ffe4e TIKA-4809: Fix /meta's spurious depth-limit flag and
envelope Content-Type (#3007)
add 84dd8827a2 TIKA-4804: complete the call in
fetchAndParseServerSideStreaming (#3008)
add cf9c2c8660 TIKA-4813 - pre-4.0.0 cleanup: clock-agnostic ParseTimeout,
nanoTime inte… (#3011)
new 5de803215b Merge remote-tracking branch 'origin/main' into
TIKA-4816-metadata-key-api
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGES.txt | 4 +
.../integration-testing/tika-eval-regression.adoc | 3 +-
.../ROOT/pages/advanced/local-vlm-server.adoc | 6 +-
.../ROOT/pages/advanced/setting-limits.adoc | 68 +-
.../configuration/parsers/external-parser.adoc | 2 +-
.../pages/configuration/parsers/tess4j-parser.adoc | 9 +-
.../pages/configuration/parsers/vlm-parsers.adoc | 6 +-
.../migration-to-4x/migrating-tika-server-4x.adoc | 12 +-
.../pages/migration-to-4x/migrating-to-4x.adoc | 11 +
docs/modules/ROOT/pages/pipes/configuration.adoc | 12 +-
docs/modules/ROOT/pages/pipes/getting-started.adoc | 5 +-
docs/modules/ROOT/pages/pipes/timeouts.adoc | 482 +++++--
docs/modules/ROOT/pages/using-tika/cli/index.adoc | 16 +-
.../ROOT/pages/using-tika/server/index.adoc | 14 +
.../src/main/java/org/apache/tika/cli/TikaCLI.java | 37 +-
.../apache/tika/cli/XmlToJsonConfigConverter.java | 21 +-
.../test/java/org/apache/tika/cli/TikaCLITest.java | 39 +-
.../tika/cli/XmlToJsonConfigConverterTest.java | 32 +-
.../xml-configs/tika-config-list-map-types.xml | 2 +-
.../java/org/apache/tika/config/ParseTimeout.java | 237 ++++
.../apache/tika/config/TikaProgressTracker.java | 82 --
.../java/org/apache/tika/config/TimeoutLimits.java | 109 +-
.../apache/tika/detect/FileCommandDetector.java | 20 +-
.../exception/EmbeddedLimitReachedException.java | 22 +-
.../tika/exception/TikaTimeoutException.java | 71 +-
.../ParsingEmbeddedDocumentExtractor.java | 45 +-
.../main/java/org/apache/tika/metadata/Audio.java | 39 +
.../tika/metadata/{XMPIdq.java => IDML.java} | 20 +-
.../java/org/apache/tika/metadata/ISO19115.java | 128 ++
.../apache/tika/metadata/{XMPPDF.java => MIF.java} | 24 +-
.../java/org/apache/tika/metadata/Metadata.java | 32 +-
.../apache/tika/metadata/TikaCoreProperties.java | 4 +
.../main/java/org/apache/tika/metadata/XMPDM.java | 11 +-
.../metadata/filter/LegacyKeyMigrationFilter.java | 95 +-
.../org/apache/tika/parser/CompositeParser.java | 16 +
.../java/org/apache/tika/parser/NetworkParser.java | 45 +-
.../java/org/apache/tika/parser/ParseRecord.java | 30 +
.../tika/parser/external/ExternalParser.java | 13 +-
.../tika/parser/external/ExternalParserConfig.java | 10 +-
.../sax/AbstractRecursiveParserWrapperHandler.java | 6 -
.../org/apache/tika/sax/DIFContentHandler.java | 150 --
.../org/apache/tika/utils/FileProcessResult.java | 39 +
.../java/org/apache/tika/utils/ProcessUtils.java | 167 ++-
.../tika/metadata/metadata-migration-3x-4x.json | 73 +
.../org/apache/tika/config/ParseTimeoutTest.java | 254 ++++
.../tika/config/TikaProgressTrackerTest.java | 103 --
.../tika/detect/FileCommandDetectorTest.java | 26 +
...arsingEmbeddedDocumentExtractorTimeoutTest.java | 276 ++++
.../org/apache/tika/metadata/KeyPrefixTest.java | 7 +-
.../metadata/MetadataInternalKeyGuardTest.java | 55 +-
.../filter/LegacyKeyMigrationFilterTest.java | 137 ++
.../apache/tika/parser/CompositeParserTest.java | 50 +
.../org/apache/tika/parser/NetworkParserTest.java | 135 ++
.../org/apache/tika/parser/mock/MockParser.java | 25 +
.../org/apache/tika/utils/ProcessUtilsTest.java | 136 ++
.../apache/tika/detect/magika/MagikaDetector.java | 18 +-
.../tika/detect/siegfried/SiegfriedDetector.java | 18 +-
.../apache/tika/pipes/grpc/TikaGrpcServerImpl.java | 1 +
.../apache/tika/pipes/grpc/TikaGrpcServerTest.java | 71 +
tika-metadata-schema/README.md | 13 +-
tika-metadata-schema/pom.xml | 35 +
.../tika/metadata/schema/MetadataKeyValidator.java | 81 +-
.../metadata/schema/MigrationTableGenerator.java | 69 +-
.../tika/metadata/schema/SchemaGenerator.java | 128 +-
.../apache/tika/metadata/metadata-key-fields.json | 88 +-
.../org/apache/tika/metadata/metadata-keys.json | 1433 +++++++++++---------
.../tika/metadata/metadata-open-namespaces.json | 61 +-
.../tika/metadata/schema/MetadataCoverageTest.java | 12 +-
.../metadata/schema/MetadataKeyValidatorTest.java | 10 +
.../schema/MetadataMigrationTableTest.java | 4 +-
.../metadata/schema/MetadataNoUnderscoreTest.java | 8 +-
.../org/apache/tika/metadata/migration-overlay.tsv | 123 ++
.../forbidden-apis-metadata-string-key-bans.txt | 32 +
tika-parent/pom.xml | 15 +
.../java/org/apache/tika/http/TikaHttpClient.java | 185 ++-
.../org/apache/tika/http/TikaHttpClientTest.java | 185 +++
.../org/apache/tika/http/TikaTestHttpServer.java | 38 +-
.../tika-parser-scientific-module/pom.xml | 19 +
.../apache/tika/parser/envi/EnviHeaderParser.java | 10 +-
.../org/apache/tika/parser/gdal/GDALParser.java | 86 +-
.../geoinfo/GeographicInformationParser.java | 108 +-
.../org/apache/tika/parser/grib/GribParser.java | 2 +-
.../java/org/apache/tika/parser/hdf/HDFParser.java | 19 +-
.../org/apache/tika/parser/isatab/ISATabUtils.java | 15 +-
.../apache/tika/parser/isatab/ISArchiveParser.java | 9 +-
.../apache/tika/parser/netcdf/NetCDFParser.java | 7 +-
.../apache/tika/parser/gdal/TestGDALParser.java | 90 +-
.../geoinfo/GeographicInformationParserTest.java | 8 +-
.../org/apache/tika/parser/hdf/HDFParserTest.java | 8 +-
.../tika/parser/isatab/ISArchiveParserTest.java | 15 +-
.../tika/parser/netcdf/NetCDFParserTest.java | 2 +-
.../tika/inference/AbstractEmbeddingFilter.java | 23 +-
.../tika/inference/ImageEmbeddingConfig.java | 10 +-
.../org/apache/tika/inference/InferenceConfig.java | 12 +-
.../tika/inference/OpenAIEmbeddingFilter.java | 7 +-
.../tika/inference/OpenAIImageEmbeddingParser.java | 23 +-
.../tika/inference/JinaEmbeddingFilterTest.java | 2 +-
.../tika/inference/OpenAIEmbeddingFilterTest.java | 2 +-
.../inference/OpenAIImageEmbeddingParserTest.java | 2 +-
.../tika-parsers-ml/tika-parser-nlp-module/pom.xml | 19 +
.../tika/parser/ctakes/CTAKESContentHandler.java | 4 +-
.../apache/tika/parser/geo/topic/GeoParser.java | 35 +-
.../tika/parser/journal/GrobidRESTParser.java | 21 +-
.../apache/tika/parser/journal/TEIDOMParser.java | 41 +-
.../apache/tika/parser/ner/NamedEntityParser.java | 11 +-
.../tika/parser/geo/topic/GeoParserTest.java | 24 +-
.../tika/parser/journal/JournalParserTest.java | 2 +-
.../org/apache/tika/parser/journal/TEITest.java | 13 +-
.../tika/parser/ner/NamedEntityParserTest.java | 12 +-
.../tika/parser/ner/nltk/NLTKNERecogniserTest.java | 2 +-
.../parser/ner/regex/RegexNERecogniserTest.java | 2 +-
.../tika/parser/ocr/tess4j/Tess4JConfig.java | 19 +-
.../tika/parser/ocr/tess4j/Tess4JParser.java | 155 ++-
.../tika/parser/ocr/tess4j/Tess4JConfigTest.java | 2 +-
.../parser/ocr/tess4j/Tess4JOCRTimeoutTest.java | 250 ++++
.../tika/parser/ocr/tess4j/Tess4JParserTest.java | 6 +-
.../resources/config-examples/tess4j-full.json | 2 +-
.../apache/tika/parser/vlm/AbstractVLMParser.java | 21 +-
.../org/apache/tika/parser/vlm/VLMOCRConfig.java | 12 +-
.../tika/parser/vlm/ClaudeVLMParserTest.java | 2 +-
.../tika/parser/vlm/GeminiVLMParserTest.java | 2 +-
.../tika/parser/vlm/OpenAIVLMParserTest.java | 4 +-
.../resources/config-examples/claude-vlm-full.json | 2 +-
.../resources/config-examples/gemini-vlm-full.json | 2 +-
.../config-examples/openai-vlm-basic.json | 2 +-
.../resources/config-examples/openai-vlm-full.json | 2 +-
.../tika/parser/RecursiveParserWrapperTest.java | 7 +-
.../config-examples/migration-full-example.json | 2 +-
.../resources/config-examples/tesseract-basic.json | 2 +-
.../resources/config-examples/tesseract-full.json | 2 +-
.../tika-parser-apple-module/pom.xml | 19 +
.../tika/parser/iwork/KeynoteContentHandler.java | 8 +-
.../tika/parser/iwork/NumbersContentHandler.java | 8 +-
.../tika/parser/iwork/PagesContentHandler.java | 37 +-
.../parser/iwork/iwana/IWork13PackageParser.java | 2 +-
.../apache/tika/parser/iwork/IWorkParserTest.java | 3 +
.../tika-parser-audiovideo-module/pom.xml | 19 +
.../org/apache/tika/parser/audio/AudioParser.java | 54 +-
.../org/apache/tika/parser/audio/MidiParser.java | 22 +-
.../java/org/apache/tika/parser/mp3/Mp3Parser.java | 13 +-
.../apache/tika/parser/mp4/TikaMp4BoxHandler.java | 9 +-
.../org/apache/tika/parser/ogg/FlacParser.java | 5 +-
.../org/apache/tika/parser/ogg/OggAudioParser.java | 27 +-
.../java/org/apache/tika/parser/ogg/OggParser.java | 12 +-
.../org/apache/tika/parser/ogg/OpusParser.java | 3 +-
.../org/apache/tika/parser/ogg/SpeexParser.java | 3 +-
.../org/apache/tika/parser/ogg/TheoraParser.java | 2 +-
.../org/apache/tika/parser/ogg/VorbisParser.java | 2 +-
.../org/apache/tika/parser/video/FLVParser.java | 9 +-
.../apache/tika/parser/audio/AudioParserTest.java | 50 +-
.../apache/tika/parser/audio/MidiParserTest.java | 6 +-
.../org/apache/tika/parser/mp3/Mp3ParserTest.java | 38 +-
.../org/apache/tika/parser/mp4/MP4ParserTest.java | 24 +-
.../apache/tika/parser/dwg/DWGParserConfig.java | 10 +-
.../org/apache/tika/parser/dwg/DWGReadParser.java | 9 +-
.../org/apache/tika/parser/dwg/DWGParserTest.java | 7 +-
.../configs/tika-config-dwgRead-Timeout.json | 2 +-
.../tika/parser/microsoft/libpst/LibPstParser.java | 7 +-
.../microsoft/libpst/LibPstParserConfig.java | 10 +-
.../tika/parser/microsoft/ExcelParserTest.java | 4 +-
.../parser/microsoft/PowerPointParserTest.java | 4 +-
.../tika/parser/microsoft/WordParserTest.java | 4 +-
.../parser/microsoft/ooxml/OOXMLDocxSAXTest.java | 4 +-
.../parser/microsoft/ooxml/OOXMLParserTest.java | 4 +-
.../parser/microsoft/ooxml/OOXMLPptxSAXTest.java | 4 +-
.../tika-parser-miscoffice-module/pom.xml | 19 +
.../apache/tika/parser/dif/DIFContentHandler.java | 9 +-
.../indesign/ContentAndMetadataExtractor.java | 3 +-
.../apache/tika/parser/indesign/IDMLParser.java | 9 +-
.../apache/tika/parser/mif/MIFContentHandler.java | 7 +-
.../org/apache/tika/parser/dif/DIFParserTest.java | 6 +-
.../tika/parser/indesign/IDMLParserTest.java | 5 +-
.../apache/tika/parser/ocr/TesseractOCRConfig.java | 16 +-
.../apache/tika/parser/ocr/TesseractOCRParser.java | 26 +-
.../tika/parser/ocr/TesseractOCRConfigTest.java | 6 +-
.../tika/parser/ocr/TesseractOCRParserTest.java | 2 +-
.../resources/configs/TIKA-3582-tesseract.json | 2 +-
.../configs/tika-config-tesseract-full.json | 2 +-
.../configs/tika-config-tesseract-partial.json | 2 +-
.../apache/tika/parser/pdf/AbstractPDF2XHTML.java | 30 +
.../tika/renderer/pdf/poppler/PopplerRenderer.java | 18 +-
.../org/apache/tika/parser/pdf/PDFParserTest.java | 95 ++
.../org/apache/tika/parser/pkg/UnrarParser.java | 9 +-
.../apache/tika/parser/strings/StringsConfig.java | 24 +-
.../apache/tika/parser/strings/StringsParser.java | 19 +-
.../tika/parser/strings/StringsConfigTest.java | 6 +-
.../configs/tika-config-strings-full.json | 2 +-
.../configs/tika-config-strings-partial.json | 2 +-
.../org/apache/tika/async/cli/PluginsWriter.java | 10 +-
.../apache/tika/async/cli/SimpleAsyncConfig.java | 16 +-
.../org/apache/tika/async/cli/TikaAsyncCLI.java | 9 +-
.../apache/tika/async/cli/AsyncCliParserTest.java | 12 +-
.../apache/tika/async/cli/AsyncProcessorTest.java | 118 ++
.../tika/async/cli/TikaConfigAsyncWriterTest.java | 35 +
.../org/apache/tika/pipes/api/PipesResult.java | 13 +-
.../tika/pipes/core/PerClientServerManager.java | 4 +-
.../org/apache/tika/pipes/core/PipesClient.java | 87 +-
.../org/apache/tika/pipes/core/PipesConfig.java | 105 +-
.../org/apache/tika/pipes/core/ServerManager.java | 4 +-
.../tika/pipes/core/SharedServerManager.java | 4 +-
.../apache/tika/pipes/core/async/AsyncEmitter.java | 7 +
.../tika/pipes/core/async/AsyncProcessor.java | 11 +-
.../tika/pipes/core/protocol/PipesMessage.java | 29 +-
.../tika/pipes/core/server/ConnectionHandler.java | 57 +-
.../apache/tika/pipes/core/server/EmitHandler.java | 40 +
.../tika/pipes/core/server/ParseHandler.java | 8 +-
.../apache/tika/pipes/core/server/PipesServer.java | 114 +-
.../tika/pipes/core/server/ServerProtocolIO.java | 29 +
.../tika/pipes/core/protocol/PipesMessageTest.java | 4 +-
.../pipes/core/server/ServerProtocolIOTest.java | 72 +
.../apache/tika/pipes/core/EmbeddedLimitsTest.java | 6 +-
.../pipes/core/PartialTimeoutPassbackTest.java | 80 ++
.../apache/tika/pipes/core/PipesClientTest.java | 181 ++-
.../configs/tika-config-timeout-lt-heartbeat.json | 4 +-
.../pipes/pipesiterator/PipesIteratorBase.java | 14 +-
.../tika/pipes/reporter/es/ESReporterConfig.java | 16 +-
.../apache/tika/pipes/es/ConfigExamplesTest.java | 18 +
.../tika/pipes/fetcher/http/HttpFetcher.java | 5 +-
.../org/apache/tika/config/TimeoutLimitsTest.java | 30 +-
.../sample-configs/vision/vlm-openai.json | 2 +-
.../server/core/resource/MetadataResource.java | 17 +-
.../server/core/resource/PipesParsingHelper.java | 2 +-
.../tika/server/standard/MetadataResourceTest.java | 46 +-
223 files changed, 7011 insertions(+), 2223 deletions(-)
create mode 100644
tika-core/src/main/java/org/apache/tika/config/ParseTimeout.java
delete mode 100644
tika-core/src/main/java/org/apache/tika/config/TikaProgressTracker.java
copy tika-core/src/main/java/org/apache/tika/metadata/{XMPIdq.java =>
IDML.java} (64%)
create mode 100644
tika-core/src/main/java/org/apache/tika/metadata/ISO19115.java
copy tika-core/src/main/java/org/apache/tika/metadata/{XMPPDF.java =>
MIF.java} (57%)
delete mode 100644
tika-core/src/main/java/org/apache/tika/sax/DIFContentHandler.java
create mode 100644
tika-core/src/test/java/org/apache/tika/config/ParseTimeoutTest.java
delete mode 100644
tika-core/src/test/java/org/apache/tika/config/TikaProgressTrackerTest.java
create mode 100644
tika-core/src/test/java/org/apache/tika/extractor/ParsingEmbeddedDocumentExtractorTimeoutTest.java
create mode 100644
tika-core/src/test/java/org/apache/tika/parser/NetworkParserTest.java
create mode 100644
tika-core/src/test/java/org/apache/tika/utils/ProcessUtilsTest.java
create mode 100644 tika-parent/forbidden-apis-metadata-string-key-bans.txt
create mode 100644
tika-parsers/tika-http-jdk/src/test/java/org/apache/tika/http/TikaHttpClientTest.java
create mode 100644
tika-parsers/tika-parsers-ml/tika-parser-tess4j-module/src/test/java/org/apache/tika/parser/ocr/tess4j/Tess4JOCRTimeoutTest.java
create mode 100644
tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/server/ServerProtocolIOTest.java
create mode 100644
tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/PartialTimeoutPassbackTest.java
