This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 21b7ba5a6c TIKA-4941: record the external DTDs and entities a
document's XML referenced (#3271)
21b7ba5a6c is described below
commit 21b7ba5a6c4673d742dbcb5e631cc9eac66faf9f
Author: Tim Allison <[email protected]>
AuthorDate: Mon Sep 28 16:46:02 2026 -0400
TIKA-4941: record the external DTDs and entities a document's XML
referenced (#3271)
---
CHANGES.txt | 5 +
.../apache/tika/metadata/TikaCoreProperties.java | 18 +++
.../org/apache/tika/parser/CompositeParser.java | 5 +-
.../java/org/apache/tika/parser/ParseRecord.java | 39 +++++-
.../org/apache/tika/sax/OfflineContentHandler.java | 16 ++-
...tentHandler.java => OfflineEntityResolver.java} | 29 ++---
.../java/org/apache/tika/utils/XMLReaderUtils.java | 13 +-
.../parser/ParseRecordExternalReferenceTest.java | 77 ++++++++++++
.../tika/parser/XmlExternalReferenceTest.java | 138 +++++++++++++++++++++
.../apache/tika/sax/OfflineEntityResolverTest.java | 80 ++++++++++++
.../apache/tika/metadata/metadata-key-fields.json | 3 +
.../org/apache/tika/metadata/metadata-keys.json | 3 +
.../parser/XmlExternalReferenceMetadataTest.java | 104 ++++++++++++++++
.../parser/odf/NSNormalizerContentHandler.java | 10 +-
14 files changed, 508 insertions(+), 32 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 1f75ba7f27..9d213462a5 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,11 @@ Release 4.2.0 - unreleased
* Retire Tika's entity expansion limit (default 20) in SAX, DOM and StAX
parsing in favor of standard Java configuration methods (TIKA-4940).
+ * Report the external DTDs and entities a document's XML referenced, which
Tika never
+ resolves: tk:xml-external-reference (up to 20 system ids) and
+ tk:xml-external-reference-count on that document's metadata, and
+ tk:xml-external-reference-embedded on a container whose embedded document
had any.
+ Every resolver Tika installs answers with an empty stream, never null
(TIKA-4941).
* Deprecate XMLReaderUtils.getXMLInputFactory() and use SAX for XFA
(TIKA-4938).
Release 4.1.0 - 9/26/2026
diff --git
a/tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java
b/tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java
index c528f3f34d..d1587b1873 100644
--- a/tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java
+++ b/tika-core/src/main/java/org/apache/tika/metadata/TikaCoreProperties.java
@@ -202,6 +202,24 @@ public interface TikaCoreProperties {
Property EMBEDDED_DEPTH_LIMIT_REACHED =
Property.reservedInternalBoolean(TIKA_META_EXCEPTION_PREFIX +
"embedded-depth-limit-reached");
+ /**
+ * External DTDs and entities this document's XML referenced, by system id
(public id
+ * when there is none). Tika never resolves them; this records what the
document would
+ * have fetched. At most {@link
org.apache.tika.parser.ParseRecord#MAX_EXTERNAL_REFERENCES}
+ * are listed; {@link #XML_EXTERNAL_REFERENCE_COUNT} is the full count.
XML that a
+ * third-party library parses with its own parser (OOXML package parts,
plists, feeds) is
+ * not seen.
+ */
+ Property XML_EXTERNAL_REFERENCE =
+ Property.reservedInternalTextBag(TIKA_META_PREFIX +
"xml-external-reference");
+
+ Property XML_EXTERNAL_REFERENCE_COUNT =
+ Property.reservedInternalInteger(TIKA_META_PREFIX +
"xml-external-reference-count");
+
+ //on the container: an embedded document recorded an XML_EXTERNAL_REFERENCE
+ Property XML_EXTERNAL_REFERENCE_EMBEDDED =
+ Property.reservedInternalBoolean(TIKA_META_PREFIX +
"xml-external-reference-embedded");
+
//total timeout exhausted mid-parse; remaining embedded docs were skipped,
not attempted
Property TASK_DEADLINE_REACHED =
Property.reservedInternalBoolean(TIKA_META_EXCEPTION_PREFIX +
"task-deadline-reached");
diff --git
a/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java
b/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java
index ba7a47f800..96cd5fa42b 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/CompositeParser.java
@@ -354,7 +354,7 @@ public class CompositeParser implements Parser {
String parserClassname = ParserUtils.getParserClassname(parser);
parserRecord.addParserClass(parserClassname);
ParserUtils.recordParserDetails(parserClassname, metadata);
- parserRecord.beforeParse();
+ parserRecord.beforeParse(metadata);
try {
parser.parse(tis, taggedHandler, metadata, context);
} catch (SecurityException e) {
@@ -410,6 +410,9 @@ public class CompositeParser implements Parser {
if (record.isTaskDeadlineReached()) {
metadata.set(TikaCoreProperties.TASK_DEADLINE_REACHED, true);
}
+ if (record.isExternalReferenceInEmbedded()) {
+ metadata.set(TikaCoreProperties.XML_EXTERNAL_REFERENCE_EMBEDDED,
true);
+ }
for (Metadata m : record.getMetadataList()) {
for (String n : m.names()) {
diff --git a/tika-core/src/main/java/org/apache/tika/parser/ParseRecord.java
b/tika-core/src/main/java/org/apache/tika/parser/ParseRecord.java
index ed5e57c3ea..a99f3e92f5 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/ParseRecord.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/ParseRecord.java
@@ -16,7 +16,9 @@
*/
package org.apache.tika.parser;
+import java.util.ArrayDeque;
import java.util.ArrayList;
+import java.util.Deque;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
@@ -25,6 +27,7 @@ import org.apache.tika.config.EmbeddedLimits;
import org.apache.tika.config.TimeoutLimits;
import org.apache.tika.config.TransientParseState;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
/**
* Use this class to store exceptions, warnings and other information
@@ -46,6 +49,8 @@ public class ParseRecord implements TransientParseState {
private static final int MAX_WARNINGS = 100;
private static final int MAX_METADATA_LIST_SIZE = 100;
+ public static final int MAX_EXTERNAL_REFERENCES = 20;
+ private static final int MAX_EXTERNAL_REFERENCE_LENGTH = 1000;
private int depth = 0;
private final Set<String> parsers = new LinkedHashSet<>();
@@ -55,6 +60,9 @@ public class ParseRecord implements TransientParseState {
private final List<String> warnings = new ArrayList<>();
private final List<Metadata> metadataList = new ArrayList<>();
+ //metadata of the documents being parsed, innermost on top
+ private final Deque<Metadata> documents = new ArrayDeque<>();
+ private boolean externalReferenceInEmbedded = false;
private boolean writeLimitReached = false;
@@ -90,12 +98,41 @@ public class ParseRecord implements TransientParseState {
return record;
}
- void beforeParse() {
+ void beforeParse(Metadata metadata) {
depth++;
+ documents.push(metadata);
}
void afterParse() {
depth--;
+ documents.pop();
+ }
+
+ /**
+ * Records an external DTD or entity reference Tika refused, on the
metadata of the
+ * document being parsed. No-op outside a {@link CompositeParser} parse.
+ */
+ public void addExternalReference(String reference) {
+ Metadata metadata = documents.peek();
+ if (metadata == null || reference == null) {
+ return;
+ }
+ Integer seen =
metadata.getInt(TikaCoreProperties.XML_EXTERNAL_REFERENCE_COUNT);
+ int count = seen == null ? 0 : seen;
+ metadata.set(TikaCoreProperties.XML_EXTERNAL_REFERENCE_COUNT, count +
1);
+ if (count < MAX_EXTERNAL_REFERENCES) {
+ String value = reference.length() > MAX_EXTERNAL_REFERENCE_LENGTH ?
+ reference.substring(0, MAX_EXTERNAL_REFERENCE_LENGTH) :
reference;
+ metadata.add(TikaCoreProperties.XML_EXTERNAL_REFERENCE, value);
+ }
+ //composite delegation pushes the same metadata again; embedded
documents bring their own
+ if (metadata != documents.peekLast()) {
+ externalReferenceInEmbedded = true;
+ }
+ }
+
+ public boolean isExternalReferenceInEmbedded() {
+ return externalReferenceInEmbedded;
}
public int getDepth() {
diff --git
a/tika-core/src/main/java/org/apache/tika/sax/OfflineContentHandler.java
b/tika-core/src/main/java/org/apache/tika/sax/OfflineContentHandler.java
index 6461e0946a..39a9c96edb 100644
--- a/tika-core/src/main/java/org/apache/tika/sax/OfflineContentHandler.java
+++ b/tika-core/src/main/java/org/apache/tika/sax/OfflineContentHandler.java
@@ -16,10 +16,11 @@
*/
package org.apache.tika.sax;
-import org.apache.commons.io.input.ClosedInputStream;
import org.xml.sax.ContentHandler;
import org.xml.sax.InputSource;
+import org.apache.tika.parser.ParseContext;
+
/**
* Content handler decorator that always returns an empty stream from the
* {@link #resolveEntity(String, String)} method to prevent potential
@@ -29,8 +30,19 @@ import org.xml.sax.InputSource;
*/
public class OfflineContentHandler extends ContentHandlerDecorator {
+ private final OfflineEntityResolver resolver;
+
public OfflineContentHandler(ContentHandler handler) {
+ this(handler, null);
+ }
+
+ /**
+ * @param context records refused external references on the document's
metadata
+ * when it holds a {@link
org.apache.tika.parser.ParseRecord}; may be null
+ */
+ public OfflineContentHandler(ContentHandler handler, ParseContext context)
{
super(handler);
+ this.resolver = new OfflineEntityResolver(context);
}
/**
@@ -39,7 +51,7 @@ public class OfflineContentHandler extends
ContentHandlerDecorator {
*/
@Override
public InputSource resolveEntity(String publicId, String systemId) {
- return new InputSource(new ClosedInputStream());
+ return resolver.resolveEntity(publicId, systemId);
}
}
diff --git
a/tika-core/src/main/java/org/apache/tika/sax/OfflineContentHandler.java
b/tika-core/src/main/java/org/apache/tika/sax/OfflineEntityResolver.java
similarity index 61%
copy from tika-core/src/main/java/org/apache/tika/sax/OfflineContentHandler.java
copy to tika-core/src/main/java/org/apache/tika/sax/OfflineEntityResolver.java
index 6461e0946a..c2f0411e18 100644
--- a/tika-core/src/main/java/org/apache/tika/sax/OfflineContentHandler.java
+++ b/tika-core/src/main/java/org/apache/tika/sax/OfflineEntityResolver.java
@@ -17,29 +17,30 @@
package org.apache.tika.sax;
import org.apache.commons.io.input.ClosedInputStream;
-import org.xml.sax.ContentHandler;
+import org.xml.sax.EntityResolver;
import org.xml.sax.InputSource;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.ParseRecord;
+
/**
- * Content handler decorator that always returns an empty stream from the
- * {@link #resolveEntity(String, String)} method to prevent potential
- * network or other external resources from being accessed by an XML parser.
- *
- * @see <a href="https://issues.apache.org/jira/browse/TIKA-185";>TIKA-185</a>
+ * Answers every external DTD or entity with an empty stream, and records what
was asked
+ * for on the document's metadata when a {@link ParseRecord} is in the context.
*/
-public class OfflineContentHandler extends ContentHandlerDecorator {
+public class OfflineEntityResolver implements EntityResolver {
- public OfflineContentHandler(ContentHandler handler) {
- super(handler);
+ private final ParseContext context;
+
+ public OfflineEntityResolver(ParseContext context) {
+ this.context = context;
}
- /**
- * Returns an empty stream. This will make an XML parser silently
- * ignore any external entities.
- */
@Override
public InputSource resolveEntity(String publicId, String systemId) {
+ ParseRecord record = context == null ? null :
context.get(ParseRecord.class);
+ if (record != null) {
+ record.addExternalReference(systemId != null ? systemId :
publicId);
+ }
return new InputSource(new ClosedInputStream());
}
-
}
diff --git a/tika-core/src/main/java/org/apache/tika/utils/XMLReaderUtils.java
b/tika-core/src/main/java/org/apache/tika/utils/XMLReaderUtils.java
index 5570fbf8d1..0b5e502517 100644
--- a/tika-core/src/main/java/org/apache/tika/utils/XMLReaderUtils.java
+++ b/tika-core/src/main/java/org/apache/tika/utils/XMLReaderUtils.java
@@ -61,6 +61,7 @@ import org.xml.sax.helpers.DefaultHandler;
import org.apache.tika.exception.TikaException;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.sax.OfflineContentHandler;
+import org.apache.tika.sax.OfflineEntityResolver;
/**
@@ -373,8 +374,8 @@ public class XMLReaderUtils implements Serializable {
}
}
- //a supplied builder never brings its own resolver along
- builder.setEntityResolver(IGNORING_SAX_ENTITY_RESOLVER);
+ //never the builder's own resolver, and record what was refused
+ builder.setEntityResolver(new OfflineEntityResolver(context));
try {
return builder.parse(is);
} finally {
@@ -411,8 +412,8 @@ public class XMLReaderUtils implements Serializable {
}
}
- //a supplied builder never brings its own resolver along
- builder.setEntityResolver(IGNORING_SAX_ENTITY_RESOLVER);
+ //never the builder's own resolver, and record what was refused
+ builder.setEntityResolver(new OfflineEntityResolver(context));
try {
return builder.parse(new InputSource(reader));
} finally {
@@ -535,7 +536,7 @@ public class XMLReaderUtils implements Serializable {
}
}
try {
- saxParser.parse(is, new OfflineContentHandler(contentHandler));
+ saxParser.parse(is, new OfflineContentHandler(contentHandler,
context));
} finally {
releaseParser(poolSAXParser);
}
@@ -573,7 +574,7 @@ public class XMLReaderUtils implements Serializable {
}
}
try {
- saxParser.parse(new InputSource(reader), new
OfflineContentHandler(contentHandler));
+ saxParser.parse(new InputSource(reader), new
OfflineContentHandler(contentHandler, context));
} finally {
releaseParser(poolSAXParser);
}
diff --git
a/tika-core/src/test/java/org/apache/tika/parser/ParseRecordExternalReferenceTest.java
b/tika-core/src/test/java/org/apache/tika/parser/ParseRecordExternalReferenceTest.java
new file mode 100644
index 0000000000..d9c6ac4607
--- /dev/null
+++
b/tika-core/src/test/java/org/apache/tika/parser/ParseRecordExternalReferenceTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+
+public class ParseRecordExternalReferenceTest {
+
+ @Test
+ public void testCapAndCount() {
+ ParseRecord record = ParseRecord.newInstance(new ParseContext());
+ Metadata m = new Metadata();
+ record.beforeParse(m);
+ for (int i = 0; i < ParseRecord.MAX_EXTERNAL_REFERENCES + 5; i++) {
+ record.addExternalReference("http://127.0.0.1:9/"; + i);
+ }
+ record.afterParse();
+ assertEquals(ParseRecord.MAX_EXTERNAL_REFERENCES,
+ m.getValues(TikaCoreProperties.XML_EXTERNAL_REFERENCE).length);
+ assertEquals(ParseRecord.MAX_EXTERNAL_REFERENCES + 5,
+ m.getInt(TikaCoreProperties.XML_EXTERNAL_REFERENCE_COUNT));
+ assertFalse(record.isExternalReferenceInEmbedded());
+ }
+
+ @Test
+ public void testLongReferenceTruncated() {
+ ParseRecord record = ParseRecord.newInstance(new ParseContext());
+ Metadata m = new Metadata();
+ record.beforeParse(m);
+ record.addExternalReference("http://127.0.0.1:9/"; + "x".repeat(5000));
+ assertEquals(1000,
m.get(TikaCoreProperties.XML_EXTERNAL_REFERENCE).length());
+ }
+
+ @Test
+ public void testOutsideAParseIsANoOp() {
+ ParseRecord record = ParseRecord.newInstance(new ParseContext());
+ record.addExternalReference("http://127.0.0.1:9/x";);
+ assertFalse(record.isExternalReferenceInEmbedded());
+ }
+
+ @Test
+ public void testNestedGoesToInnerAndFlagsEmbedded() {
+ ParseRecord record = ParseRecord.newInstance(new ParseContext());
+ Metadata outer = new Metadata();
+ Metadata inner = new Metadata();
+ record.beforeParse(outer);
+ record.beforeParse(inner);
+ record.addExternalReference("http://127.0.0.1:9/x";);
+ record.afterParse();
+ record.afterParse();
+ assertNull(outer.get(TikaCoreProperties.XML_EXTERNAL_REFERENCE));
+ assertEquals("http://127.0.0.1:9/x";,
inner.get(TikaCoreProperties.XML_EXTERNAL_REFERENCE));
+ assertTrue(record.isExternalReferenceInEmbedded());
+ }
+}
diff --git
a/tika-core/src/test/java/org/apache/tika/parser/XmlExternalReferenceTest.java
b/tika-core/src/test/java/org/apache/tika/parser/XmlExternalReferenceTest.java
new file mode 100644
index 0000000000..99014c175f
--- /dev/null
+++
b/tika-core/src/test/java/org/apache/tika/parser/XmlExternalReferenceTest.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import org.junit.jupiter.api.Test;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.HttpHeaders;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.mime.MediaTypeRegistry;
+import org.apache.tika.utils.XMLReaderUtils;
+
+/**
+ * Refused external references land on the metadata of the document whose XML
made them,
+ * and the container is flagged when an embedded document made any.
+ */
+public class XmlExternalReferenceTest {
+
+ private static final String DTD = "file:///couldnt_possibly_exist/a.dtd";
+ private static final String ENTITY = "http://127.0.0.1:9/e.txt";;
+ private static final String XML = "<!DOCTYPE r SYSTEM \"" + DTD + "\"
[<!ENTITY e SYSTEM \"" +
+ ENTITY + "\">]><r>&e;</r>";
+
+ // parses XML through the two chokepoints, and can parse a nested
"embedded" document
+ private static class XmlParser implements Parser {
+ private final boolean dom;
+ private Parser embedded;
+ private byte[] embeddedBytes;
+ private Metadata innerMetadata;
+
+ XmlParser(boolean dom) {
+ this.dom = dom;
+ }
+
+ @Override
+ public Set<MediaType> getSupportedTypes(ParseContext context) {
+ return Collections.singleton(MediaType.APPLICATION_XML);
+ }
+
+ @Override
+ public void parse(TikaInputStream stream, ContentHandler handler,
Metadata metadata,
+ ParseContext context) throws IOException,
SAXException, TikaException {
+ if (dom) {
+ XMLReaderUtils.buildDOM(stream, context);
+ } else {
+ XMLReaderUtils.parseSAX(stream, new DefaultHandler(), context);
+ }
+ if (embedded != null) {
+ innerMetadata = new Metadata();
+ innerMetadata.set(TikaCoreProperties.RESOURCE_NAME_KEY,
"inner.xml");
+ innerMetadata.set(HttpHeaders.CONTENT_TYPE, "application/xml");
+ try (TikaInputStream tis = TikaInputStream.get(embeddedBytes))
{
+ embedded.parse(tis, new DefaultHandler(), innerMetadata,
context);
+ }
+ }
+ }
+ }
+
+ private static Metadata parse(XmlParser xmlParser, String xml) throws
Exception {
+ CompositeParser composite = new
CompositeParser(MediaTypeRegistry.getDefaultRegistry(),
+ List.of(xmlParser));
+ Metadata metadata = new Metadata();
+ metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "outer.xml");
+ metadata.set(HttpHeaders.CONTENT_TYPE, "application/xml");
+ try (TikaInputStream tis =
TikaInputStream.get(xml.getBytes(StandardCharsets.UTF_8))) {
+ composite.parse(tis, new DefaultHandler(), metadata, new
ParseContext());
+ }
+ return metadata;
+ }
+
+ @Test
+ public void testSaxRecordsOnDocument() throws Exception {
+ Metadata m = parse(new XmlParser(false), XML);
+ assertArrayEquals(new String[]{DTD, ENTITY},
+ m.getValues(TikaCoreProperties.XML_EXTERNAL_REFERENCE));
+ assertEquals(2,
m.getInt(TikaCoreProperties.XML_EXTERNAL_REFERENCE_COUNT));
+ assertNull(m.get(TikaCoreProperties.XML_EXTERNAL_REFERENCE_EMBEDDED));
+ }
+
+ @Test
+ public void testDomRecordsOnDocument() throws Exception {
+ Metadata m = parse(new XmlParser(true), XML);
+ assertTrue(m.getInt(TikaCoreProperties.XML_EXTERNAL_REFERENCE_COUNT)
>= 1);
+ assertEquals(DTD,
m.getValues(TikaCoreProperties.XML_EXTERNAL_REFERENCE)[0]);
+ }
+
+ @Test
+ public void testCleanDocumentRecordsNothing() throws Exception {
+ Metadata m = parse(new XmlParser(false), "<r>plain</r>");
+ assertNull(m.get(TikaCoreProperties.XML_EXTERNAL_REFERENCE));
+ assertNull(m.get(TikaCoreProperties.XML_EXTERNAL_REFERENCE_COUNT));
+ }
+
+ @Test
+ public void testEmbeddedRecordsOnInnerAndFlagsOuter() throws Exception {
+ XmlParser outer = new XmlParser(false);
+ outer.embedded = new
CompositeParser(MediaTypeRegistry.getDefaultRegistry(),
+ List.of(new XmlParser(false)));
+ outer.embeddedBytes = XML.getBytes(StandardCharsets.UTF_8);
+ Metadata m = parse(outer, "<r>plain outer</r>");
+ assertNull(m.get(TikaCoreProperties.XML_EXTERNAL_REFERENCE));
+ assertEquals("true",
m.get(TikaCoreProperties.XML_EXTERNAL_REFERENCE_EMBEDDED));
+ assertArrayEquals(new String[]{DTD, ENTITY},
+
outer.innerMetadata.getValues(TikaCoreProperties.XML_EXTERNAL_REFERENCE));
+
assertNull(outer.innerMetadata.get(TikaCoreProperties.XML_EXTERNAL_REFERENCE_EMBEDDED));
+ }
+}
diff --git
a/tika-core/src/test/java/org/apache/tika/sax/OfflineEntityResolverTest.java
b/tika-core/src/test/java/org/apache/tika/sax/OfflineEntityResolverTest.java
new file mode 100644
index 0000000000..814cdf0c5f
--- /dev/null
+++ b/tika-core/src/test/java/org/apache/tika/sax/OfflineEntityResolverTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.sax;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+import org.xml.sax.EntityResolver;
+import org.xml.sax.InputSource;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.utils.XMLReaderUtils;
+
+/**
+ * A resolver that returns null, or a source with no stream, hands the id back
to the
+ * parser to open (CVE-2025-66516). Every resolver Tika installs must answer
every
+ * input shape with a source that carries an empty stream.
+ */
+public class OfflineEntityResolverTest {
+
+ private static final String[][] IDS = {
+ {null, null},
+ {"-//W3C//DTD XHTML 1.0//EN", null},
+ {null, "http://127.0.0.1:9/x.dtd"},
+ {null, "file:///etc/passwd"},
+ {null, "relative.dtd"},
+ {null, ""},
+ {"-//X//Y//EN", "https://example.invalid/y.dtd"},
+ };
+
+ @Test
+ public void testInstalledResolversAlwaysAnswerWithAnEmptyStream() throws
Exception {
+ List<EntityResolver> resolvers = List.of(
+ new OfflineEntityResolver(null),
+ new OfflineEntityResolver(new ParseContext()),
+ new OfflineContentHandler(new DefaultHandler()),
+ new OfflineContentHandler(new DefaultHandler(), new
ParseContext()),
+ XMLReaderUtils.getXMLReader().getEntityResolver());
+ for (EntityResolver resolver : resolvers) {
+ for (String[] id : IDS) {
+ InputSource source = resolver.resolveEntity(id[0], id[1]);
+ String label = resolver.getClass().getSimpleName() + " for " +
id[0] + ", " + id[1];
+ assertNotNull(source, label + " returned null");
+ assertNull(source.getSystemId(), label + " carries a system
id");
+ assertTrue(source.getByteStream() != null ||
source.getCharacterStream() != null,
+ label + " carries no stream");
+ assertEquals(-1, source.getByteStream() != null ?
source.getByteStream().read() :
+ source.getCharacterStream().read(), label + " stream
is not empty");
+ }
+ }
+ }
+
+ @Test
+ public void testEmptyStreamTwiceOverIsStillEmpty() throws IOException {
+ InputSource source = new
OfflineEntityResolver(null).resolveEntity(null, "x.dtd");
+ assertEquals(-1, source.getByteStream().read());
+ assertEquals(-1, source.getByteStream().read(new byte[16]));
+ }
+}
diff --git
a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-key-fields.json
b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-key-fields.json
index e1633faa21..3f57665d29 100644
---
a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-key-fields.json
+++
b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-key-fields.json
@@ -580,6 +580,9 @@
{"class":"org.apache.tika.metadata.TikaCoreProperties","field":"VERSION_COUNT","key":"tk:version-count"},
{"class":"org.apache.tika.metadata.TikaCoreProperties","field":"VERSION_NUMBER","key":"tk:version-number"},
{"class":"org.apache.tika.metadata.TikaCoreProperties","field":"WRITE_LIMIT_REACHED","key":"tk:exception:write-limit-reached"},
+
{"class":"org.apache.tika.metadata.TikaCoreProperties","field":"XML_EXTERNAL_REFERENCE","key":"tk:xml-external-reference"},
+
{"class":"org.apache.tika.metadata.TikaCoreProperties","field":"XML_EXTERNAL_REFERENCE_COUNT","key":"tk:xml-external-reference-count"},
+
{"class":"org.apache.tika.metadata.TikaCoreProperties","field":"XML_EXTERNAL_REFERENCE_EMBEDDED","key":"tk:xml-external-reference-embedded"},
{"class":"org.apache.tika.metadata.TikaPagedText","field":"PAGE_NUMBER","key":"tk:page:number"},
{"class":"org.apache.tika.metadata.TikaPagedText","field":"PAGE_NUMBERS","key":"tk:page:numbers"},
{"class":"org.apache.tika.metadata.TikaPagedText","field":"PAGE_ROTATION","key":"tk:page:rotation"},
diff --git
a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json
b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json
index 284fed451e..4dc423e946 100644
---
a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json
+++
b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json
@@ -635,6 +635,9 @@
{"key":"tk:version-count","namespace":"tk","valueType":"INTEGER","cardinality":"SIMPLE","module":"tika-core"},
{"key":"tk:version-number","namespace":"tk","valueType":"INTEGER","cardinality":"SIMPLE","module":"tika-core"},
{"key":"tk:warn:truncated-metadata","namespace":"tk","valueType":"BOOLEAN","cardinality":"SIMPLE","module":"tika-core"},
+
{"key":"tk:xml-external-reference","namespace":"tk","valueType":"TEXT","cardinality":"BAG","module":"tika-core"},
+
{"key":"tk:xml-external-reference-count","namespace":"tk","valueType":"INTEGER","cardinality":"SIMPLE","module":"tika-core"},
+
{"key":"tk:xml-external-reference-embedded","namespace":"tk","valueType":"BOOLEAN","cardinality":"SIMPLE","module":"tika-core"},
{"key":"video:bitrate","namespace":"video","valueType":"INTEGER","cardinality":"SIMPLE","module":"tika-core"},
{"key":"video:fourcc","namespace":"video","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
{"key":"video:frame-rate","namespace":"video","valueType":"REAL","cardinality":"SIMPLE","module":"tika-core"},
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/XmlExternalReferenceMetadataTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/XmlExternalReferenceMetadataTest.java
new file mode 100644
index 0000000000..e13d87e841
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/XmlExternalReferenceMetadataTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+
+/**
+ * The document parsers report the external DTDs and entities a file would
have fetched.
+ */
+public class XmlExternalReferenceMetadataTest extends XMLTestBase {
+
+ private static final String DTD = "file:///couldnt_possibly_exist/xxe.dtd";
+ private static final byte[] DOCTYPE =
+ ("<!DOCTYPE roottag SYSTEM \"" + DTD +
"\">").getBytes(StandardCharsets.UTF_8);
+
+ @Test
+ public void testXmlFile() throws Exception {
+ byte[] injected = injectXML("<?xml
version=\"1.0\"?><r>text</r>".getBytes(StandardCharsets.UTF_8),
+ DOCTYPE);
+ Metadata m = getXML(TikaInputStream.get(injected), AUTO_DETECT_PARSER,
new Metadata()).metadata;
+ assertEquals(DTD, m.get(TikaCoreProperties.XML_EXTERNAL_REFERENCE));
+ assertEquals(1,
m.getInt(TikaCoreProperties.XML_EXTERNAL_REFERENCE_COUNT));
+ assertNull(m.get(TikaCoreProperties.XML_EXTERNAL_REFERENCE_EMBEDDED));
+ }
+
+ @Test
+ public void testCleanXmlFile() throws Exception {
+ Metadata m =
getXML(TikaInputStream.get("<r>text</r>".getBytes(StandardCharsets.UTF_8)),
+ AUTO_DETECT_PARSER, new Metadata()).metadata;
+ assertNull(m.get(TikaCoreProperties.XML_EXTERNAL_REFERENCE));
+ assertNull(m.get(TikaCoreProperties.XML_EXTERNAL_REFERENCE_COUNT));
+ }
+
+ // EPUB parts are parsed inline, so the container itself carries the ids
+ @Test
+ public void testEpubPartsLandOnContainer() throws Exception {
+ Path injected;
+ try (TikaInputStream tis = TikaInputStream.get(
+
getClass().getResourceAsStream("/test-documents/testEPUB.epub"))) {
+ injected = injectZippedXMLs(tis.getPath(), DOCTYPE);
+ }
+ try {
+ Metadata container = getRecursiveMetadata(injected).get(0);
+ assertEquals(DTD,
container.get(TikaCoreProperties.XML_EXTERNAL_REFERENCE));
+
assertTrue(container.getInt(TikaCoreProperties.XML_EXTERNAL_REFERENCE_COUNT) >=
1);
+
assertNull(container.get(TikaCoreProperties.XML_EXTERNAL_REFERENCE_EMBEDDED));
+ } finally {
+ Files.delete(injected);
+ }
+ }
+
+ // an XML file inside a zip is an embedded document: it carries the ids,
the zip is flagged
+ @Test
+ public void testEmbeddedXmlFlagsContainer() throws Exception {
+ Path zip = Files.createTempFile("tika-xxe-", ".zip");
+ try (ZipOutputStream out = new
ZipOutputStream(Files.newOutputStream(zip))) {
+ out.putNextEntry(new ZipEntry("inner.xml"));
+ out.write(injectXML("<?xml
version=\"1.0\"?><r>text</r>".getBytes(StandardCharsets.UTF_8),
+ DOCTYPE));
+ out.closeEntry();
+ }
+ try {
+ List<Metadata> all = getRecursiveMetadata(zip);
+ assertEquals(2, all.size());
+ Metadata container = all.get(0);
+ Metadata inner = all.get(1);
+ assertEquals("true",
container.get(TikaCoreProperties.XML_EXTERNAL_REFERENCE_EMBEDDED));
+
assertNull(container.get(TikaCoreProperties.XML_EXTERNAL_REFERENCE));
+ assertEquals(DTD,
inner.get(TikaCoreProperties.XML_EXTERNAL_REFERENCE));
+ assertEquals(1,
inner.getInt(TikaCoreProperties.XML_EXTERNAL_REFERENCE_COUNT));
+ } finally {
+ Files.delete(zip);
+ }
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/NSNormalizerContentHandler.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/NSNormalizerContentHandler.java
index 48cf14b2e8..1496f71e33 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/NSNormalizerContentHandler.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/NSNormalizerContentHandler.java
@@ -18,7 +18,6 @@ package org.apache.tika.parser.odf;
import java.io.IOException;
import java.io.StringReader;
-import java.util.Locale;
import org.xml.sax.Attributes;
import org.xml.sax.ContentHandler;
@@ -40,7 +39,6 @@ public class NSNormalizerContentHandler extends
ContentHandlerDecorator {
private static final String NEW_NS =
"urn:oasis:names:tc:opendocument:xmlns:";
- private static final String DTD_PUBLIC_ID = "-//OpenOffice.org//DTD
OfficeDocument 1.0//EN";
public NSNormalizerContentHandler(ContentHandler handler) {
super(handler);
@@ -83,12 +81,8 @@ public class NSNormalizerContentHandler extends
ContentHandlerDecorator {
@Override
public InputSource resolveEntity(String publicId, String systemId)
throws IOException, SAXException {
- if ((systemId != null &&
systemId.toLowerCase(Locale.ROOT).endsWith(".dtd")) ||
- DTD_PUBLIC_ID.equals(publicId)) {
- return new InputSource(new StringReader(""));
- } else {
- return super.resolveEntity(publicId, systemId);
- }
+ //never null: a null answer lets the parser resolve the id itself
+ return new InputSource(new StringReader(""));
}
}
