This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 5b78cd84bc TIKA-4873 - give embedded documents their size (#3117)
5b78cd84bc is described below
commit 5b78cd84bce98bfb9e8eea2f9b942ad0a4aeb2a0
Author: Dominik Schmidt <[email protected]>
AuthorDate: Thu Sep 3 02:58:32 2026 +0200
TIKA-4873 - give embedded documents their size (#3117)
A parser builds the metadata of an embedded document beside the stream it
hands over, so a length the stream already knows never reached the metadata
a client sees: the raw camera previews, the audio cover art and others came
back with a type and a name but nothing about how many bytes they are.
ParsingEmbeddedDocumentExtractor now fills Content-Length where the field is
blank and the stream knows its length. hasLength() answers from the source,
so nothing is ever spooled to measure it, and a stream that would have to be
read to be measured is left alone. A length the parser set stands: it
describes the item, which need not be the whole of the stream.
That covers every file or byte array backed embedded stream. Where the
length is read from the file rather than known to the stream, the parser
still has to say so: RawTiffParser takes the preview length from the IFD,
and CoverArt hands the picture metadata to TikaInputStream, which puts the
length there rather than into an object nobody sees.
TIKA-4106 approaches the same gap through the digester, which fills the
length in for callers who ask for digests, and only once the stream has been
read.
---
CHANGES.txt | 6 ++
.../ParsingEmbeddedDocumentExtractor.java | 9 +++
.../tika/extractor/EmbeddedContentLengthTest.java | 84 ++++++++++++++++++++++
.../org/apache/tika/parser/audio/CoverArt.java | 4 +-
.../apache/tika/parser/image/RawTiffParser.java | 4 ++
.../tika/parser/image/RawTiffParserTest.java | 4 ++
6 files changed, 110 insertions(+), 1 deletion(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 0d7c4604cd..c11045fde9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,11 @@
Release 4.1.0 - unreleased
+ * Embedded documents carry their size: ParsingEmbeddedDocumentExtractor
+ sets Content-Length from the stream where the stream knows it and the
+ parser did not say, which never spools to measure one, and the raw
+ camera previews and the audio cover art set the length they read from
+ the file (TIKA-4873).
+
* AVIF images are parsed rather than only detected: HeifParser accepts
image/avif, which is the same ISO-BMFF container, so dimensions, EXIF
and XMP come out of it the way they do for HEIC (TIKA-4870).
diff --git
a/tika-core/src/main/java/org/apache/tika/extractor/ParsingEmbeddedDocumentExtractor.java
b/tika-core/src/main/java/org/apache/tika/extractor/ParsingEmbeddedDocumentExtractor.java
index c94383fe49..3b66a222af 100644
---
a/tika-core/src/main/java/org/apache/tika/extractor/ParsingEmbeddedDocumentExtractor.java
+++
b/tika-core/src/main/java/org/apache/tika/extractor/ParsingEmbeddedDocumentExtractor.java
@@ -32,6 +32,7 @@ import
org.apache.tika.exception.EmbeddedLimitReachedException;
import org.apache.tika.exception.EncryptedDocumentException;
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.DelegatingParser;
@@ -42,6 +43,7 @@ import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.sax.EmbeddedContentHandler;
import org.apache.tika.sax.SAXOutputConfig;
import org.apache.tika.sax.XHTMLBalancingHandler;
+import org.apache.tika.utils.StringUtils;
/**
* Helper class for parsers of package archives or other compound document
@@ -182,6 +184,13 @@ public class ParsingEmbeddedDocumentExtractor implements
EmbeddedDocumentExtract
parseRecord.incrementEmbeddedCount();
}
+ // A parser builds the metadata beside the stream, so a length the
stream
+ // knows is often missing from it. hasLength() answers from the source,
+ // without spooling a stream to measure it (TIKA-4873).
+ if (StringUtils.isBlank(metadata.get(HttpHeaders.CONTENT_LENGTH)) &&
tis.hasLength()) {
+ metadata.set(HttpHeaders.CONTENT_LENGTH,
Long.toString(tis.getLength()));
+ }
+
if (outputHtml) {
AttributesImpl attributes = new AttributesImpl();
attributes.addAttribute("", "class", "class", "CDATA",
"package-entry");
diff --git
a/tika-core/src/test/java/org/apache/tika/extractor/EmbeddedContentLengthTest.java
b/tika-core/src/test/java/org/apache/tika/extractor/EmbeddedContentLengthTest.java
new file mode 100644
index 0000000000..725884fbce
--- /dev/null
+++
b/tika-core/src/test/java/org/apache/tika/extractor/EmbeddedContentLengthTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.extractor;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.io.InputStream;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.HttpHeaders;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.sax.BodyContentHandler;
+
+/**
+ * A parser builds the metadata of an embedded document beside the stream it
+ * hands over, so a length the stream knows was missing from what a client
+ * sees (TIKA-4873).
+ */
+public class EmbeddedContentLengthTest {
+
+ private static final byte[] CONTENT = "0123456789".getBytes(UTF_8);
+
+ @Test
+ public void testTheLengthComesFromTheStream() throws Exception {
+ ParseContext context = new ParseContext();
+ Metadata metadata = new Metadata();
+ try (TikaInputStream tis = TikaInputStream.get(CONTENT)) {
+ EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(context)
+ .parseEmbedded(tis, new BodyContentHandler(), metadata,
context, false);
+ }
+ assertEquals("10", metadata.get(HttpHeaders.CONTENT_LENGTH));
+ }
+
+ /**
+ * What the parser says stands: it describes the item, which need not be
+ * the whole of the stream it happens to hand over.
+ */
+ @Test
+ public void testADeclaredLengthIsKept() throws Exception {
+ ParseContext context = new ParseContext();
+ Metadata metadata = new Metadata();
+ metadata.set(HttpHeaders.CONTENT_LENGTH, "7");
+ try (TikaInputStream tis = TikaInputStream.get(CONTENT)) {
+ EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(context)
+ .parseEmbedded(tis, new BodyContentHandler(), metadata,
context, false);
+ }
+ assertEquals("7", metadata.get(HttpHeaders.CONTENT_LENGTH));
+ }
+
+ /**
+ * A stream that would have to be read to be measured is left alone: the
+ * length is worth less than a copy of the content to find it.
+ */
+ @Test
+ public void testAStreamOfUnknownLengthIsNotMeasured() throws Exception {
+ ParseContext context = new ParseContext();
+ Metadata metadata = new Metadata();
+ try (InputStream bare = new java.io.ByteArrayInputStream(CONTENT);
+ TikaInputStream tis = TikaInputStream.get(bare)) {
+ EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(context)
+ .parseEmbedded(tis, new BodyContentHandler(), metadata,
context, false);
+ }
+ assertNull(metadata.get(HttpHeaders.CONTENT_LENGTH));
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/audio/CoverArt.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/audio/CoverArt.java
index ebdc8620c2..483735380d 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/audio/CoverArt.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/audio/CoverArt.java
@@ -146,7 +146,9 @@ public final class CoverArt {
ID3Tags.PICTURE_TYPES[picture.type()]);
}
if (extractor.shouldParseEmbedded(pictureMetadata, context)) {
- try (TikaInputStream pictureStream =
TikaInputStream.get(picture.data())) {
+ //the metadata takes the length of the picture from the stream
+ try (TikaInputStream pictureStream =
+ TikaInputStream.get(picture.data(), pictureMetadata)) {
extractor.parseEmbedded(pictureStream, xhtml,
pictureMetadata, context, true);
}
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/RawTiffParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/RawTiffParser.java
index 774aa3a8a9..44ca2202f8 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/RawTiffParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/RawTiffParser.java
@@ -193,6 +193,10 @@ public class RawTiffParser extends TiffParser {
for (Preview preview : previews) {
Metadata previewMetadata = Metadata.newInstance(context);
previewMetadata.set(HttpHeaders.CONTENT_TYPE, JPEG_MIME);
+ //the IFD gives the length, so nothing has to read the preview
+ //to find out how long it is (TIKA-4873)
+ previewMetadata.set(HttpHeaders.CONTENT_LENGTH,
+ Long.toString(preview.length()));
if (count == 0) {
previewMetadata.set(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE,
TikaCoreProperties.EmbeddedResourceType.THUMBNAIL.toString());
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/RawTiffParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/RawTiffParserTest.java
index 8a8138d757..deb7b0f17c 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/RawTiffParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/RawTiffParserTest.java
@@ -18,6 +18,7 @@ package org.apache.tika.parser.image;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.Arrays;
import java.util.List;
@@ -67,6 +68,9 @@ public class RawTiffParserTest extends TikaTest {
assertEquals(type.toString(),
preview.get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE));
assertEquals(Integer.toString(width), preview.get(TIFF.IMAGE_WIDTH));
assertEquals(Integer.toString(height), preview.get(TIFF.IMAGE_LENGTH));
+ //the IFD knows how long the preview is, so the metadata says so
+ assertTrue(Long.parseLong(preview.get(HttpHeaders.CONTENT_LENGTH)) > 0,
+ preview.get(HttpHeaders.CONTENT_LENGTH));
}
@Test
