This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch branch_3x
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/branch_3x by this push:
new 7bcd3d7827 Dependency updates branch 3x (#3629)
7bcd3d7827 is described below
commit 7bcd3d782791a7412bdb6defab18a342b60fac1a
Author: Tim Allison <[email protected]>
AuthorDate: Tue Nov 17 09:02:38 2026 -0500
Dependency updates branch 3x (#3629)
* Dependency updates for branch_3x; juniversalchardet 2.5.2
* TIKA-4962 follow-up: gate case-insensitive html magic on markup-like file
starts
---
.mvn/wrapper/maven-wrapper.properties | 2 +-
CHANGES.txt | 3 ++
tika-app/pom.xml | 2 +-
.../org/apache/tika/mime/tika-mimetypes.xml | 50 ++++++++++++++++++++--
.../org/apache/tika/mime/MimeDetectionTest.java | 43 +++++++++++++++++++
tika-parent/pom.xml | 18 ++++----
.../tika/parser/txt/UniversalEncodingListener.java | 4 ++
.../org/apache/tika/parser/txt/TXTParserTest.java | 16 +++++++
tika-server/pom.xml | 2 +-
tika-server/tika-server-core/pom.xml | 2 +-
10 files changed, 126 insertions(+), 16 deletions(-)
diff --git a/.mvn/wrapper/maven-wrapper.properties
b/.mvn/wrapper/maven-wrapper.properties
index bb58b3c263..0d710c43f2 100644
--- a/.mvn/wrapper/maven-wrapper.properties
+++ b/.mvn/wrapper/maven-wrapper.properties
@@ -1,3 +1,3 @@
wrapperVersion=3.3.4
distributionType=only-script
-distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.10.0/apache-maven-3.10.0-bin.zip
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.10.1/apache-maven-3.10.1-bin.zip
diff --git a/CHANGES.txt b/CHANGES.txt
index ee7e520d7b..70eeff45dc 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,8 @@
Release 3.4.0 - ???
+ * UTF-8 text that was misdetected as GB18030 is now detected as UTF-8
+ (juniversalchardet 2.5.2).
+
* The http fetcher, the Solr and OpenSearch emitters and the Solr pipes
iterator no longer
send Authorization or Cookie headers, or answer a 401 with the configured
password,
after a redirect to another host, port or scheme. That includes http to
https, so use
diff --git a/tika-app/pom.xml b/tika-app/pom.xml
index 7c21c9a15d..3b201072d1 100644
--- a/tika-app/pom.xml
+++ b/tika-app/pom.xml
@@ -243,7 +243,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
- <version>3.6.1</version>
+ <version>3.8.0</version>
<executions>
<execution>
<phase>package</phase>
diff --git
a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
index 0c075a2271..2f12d5e793 100644
--- a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
+++ b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml
@@ -7752,8 +7752,45 @@
<match value="<head" type="string" offset="0:64"/>
<match value="<TITLE" type="string" offset="0:64"/>
<match value="<title" type="string" offset="0:64"/>
- <!-- the tag must end: <htmlPath> in an XML file is not HTML -->
- <match value="(?i)<html[\\s>]" type="regex" offset="0:128"/>
+ <!-- the tag must end: <htmlPath> in an XML file is not HTML. Short
strings
+ rather than a regex: a pattern longer than xhtml's "<html xmlns="
would
+ sort ahead of it at this priority. -->
+ <match value="<html " type="string" offset="0:128"/>
+ <match value="<html>" type="string" offset="0:128"/>
+ <match value="<html\n" type="string" offset="0:128"/>
+ <match value="<html\r" type="string" offset="0:128"/>
+ <match value="<html\x09" type="string" offset="0:128"/>
+ <match value="<HTML " type="string" offset="0:64"/>
+ <match value="<HTML>" type="string" offset="0:64"/>
+ <match value="<HTML\n" type="string" offset="0:64"/>
+ <match value="<HTML\r" type="string" offset="0:64"/>
+ <match value="<HTML\x09" type="string" offset="0:64"/>
+ <!-- past 64 bytes, uppercase only when the file starts like markup (see
below) -->
+ <match value="<HTML " type="string" offset="64:128">
+ <match value="<[^?]" type="regex" offset="0"/>
+ <match value="\\s" type="regex" offset="0"/>
+ <match value="0xEFBBBF" type="string" offset="0"/>
+ </match>
+ <match value="<HTML>" type="string" offset="64:128">
+ <match value="<[^?]" type="regex" offset="0"/>
+ <match value="\\s" type="regex" offset="0"/>
+ <match value="0xEFBBBF" type="string" offset="0"/>
+ </match>
+ <match value="<HTML\n" type="string" offset="64:128">
+ <match value="<[^?]" type="regex" offset="0"/>
+ <match value="\\s" type="regex" offset="0"/>
+ <match value="0xEFBBBF" type="string" offset="0"/>
+ </match>
+ <match value="<HTML\r" type="string" offset="64:128">
+ <match value="<[^?]" type="regex" offset="0"/>
+ <match value="\\s" type="regex" offset="0"/>
+ <match value="0xEFBBBF" type="string" offset="0"/>
+ </match>
+ <match value="<HTML\x09" type="string" offset="64:128">
+ <match value="<[^?]" type="regex" offset="0"/>
+ <match value="\\s" type="regex" offset="0"/>
+ <match value="0xEFBBBF" type="string" offset="0"/>
+ </match>
</magic>
<magic priority="20">
<!-- Lower priority match for <html anywhere near the top of the file -->
@@ -7763,7 +7800,14 @@
MimeTypes#getMinLength() bytes. It should also only start after
the higher priority "start of file" one above
-->
- <match value="(?i)<html[\\s>]" type="regex" offset="128:8192"/>
+ <match value="<html[\\s>]" type="regex" offset="128:8192"/>
+ <!-- other cases only when the file starts like markup, so that code
+ building "<HTML>" strings or heredocs is not taken for html -->
+ <match value="(?i)<html[\\s>]" type="regex" offset="128:8192">
+ <match value="<[^?]" type="regex" offset="0"/>
+ <match value="\\s" type="regex" offset="0"/>
+ <match value="0xEFBBBF" type="string" offset="0"/>
+ </match>
</magic>
<glob pattern="*.html"/>
<glob pattern="*.htm"/>
diff --git
a/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
b/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
index acbc583dd5..2c75a2e258 100644
--- a/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
+++ b/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
@@ -112,6 +112,49 @@ public class MimeDetectionTest {
}
}
+ /** An uppercase html tag past the first 128 bytes counts only if the file
starts like markup. */
+ @Test
+ public void testDeepUppercaseHtml() throws Exception {
+ String pad = "x = 1;\n".repeat(30);
+ String js = "var w = window.open();\n" + pad +
"w.document.write(\"<HTML><BODY>\");\n";
+ assertEquals(MediaType.TEXT_PLAIN, detect(js, null));
+ assertEquals(MediaType.text("javascript"), detect(js, "popup.js"));
+
+ String perl = "package Foo;\n" + pad + "print
<<EOF;\n<HTML>\n<HEAD><TITLE>x</TITLE>\n";
+ assertEquals(MediaType.text("x-perl"), detect(perl, "Foo.pm"));
+
+ String php = "<?\n" + pad + "?>\n<HTML>\n<TITLE>x</TITLE>\n";
+ assertEquals(MediaType.text("x-php"), detect(php, "index.php"));
+
+ // within the first 128 bytes too
+ String perlNear = "#\n# Sort hosts\n#\n&make_info();\n\nprint CLIENT
<<EOF;\n" +
+ " ".repeat(5) + "<HTML>\n<HEAD><TITLE>x</TITLE>\n";
+ assertEquals(MediaType.text("x-perl"), detect(perlNear,
"satan_counts.pl"));
+ String phpNear = "<? include(\"dbconnect.php3\"); ?>\n<? $sitename =
doins_var(\"name\");\n" +
+ "print \"<HTML> <HEAD><TITLE>About</TITLE> </HEAD>\";";
+ assertEquals(MediaType.text("x-php"), detect(phpNear, "about.php3"));
+ assertEquals(MediaType.TEXT_HTML,
+ detect("<!-- top.src START -->" + " ".repeat(60) +
"<HTML>\n<HEAD>", null));
+
+ String[] pages = {
+ "\r\n\t".repeat(100) + "<HTML>\n<BODY>x</BODY></HTML>",
+ "<SCRIPT SRC=\"/cookie.js\"></SCRIPT>\n" + " ".repeat(200) +
"<HTML>\n<BODY>x</BODY>",
+ "" + " ".repeat(200) + "<HTML>\n<BODY>x</BODY>"
+ };
+ for (String p : pages) {
+ assertEquals(MediaType.TEXT_HTML, detect(p, null), p);
+ }
+ }
+
+ /** A bare html tag must not outrank xhtml's "html xmlns=" match, and a
.html name still wins. */
+ @Test
+ public void testXhtmlNamespaceWithoutDoctype() throws Exception {
+ String xhtml = "<!DOCTYPE PUBLIC \"-//W3C//DTD HTML 4.01//EN\">\n" +
+ "<html xmlns=\"http://www.w3.org/1999/xhtml\">\n<head><meta
charset=utf-8></head>";
+ assertEquals(MediaType.application("xhtml+xml"), detect(xhtml, null));
+ assertEquals(MediaType.TEXT_HTML, detect(xhtml, "page.html"));
+ }
+
private static MediaType detect(String content, String name) throws
IOException {
Metadata metadata = new Metadata();
if (name != null) {
diff --git a/tika-parent/pom.xml b/tika-parent/pom.xml
index 6262f69a27..5f82f2bec8 100644
--- a/tika-parent/pom.xml
+++ b/tika-parent/pom.xml
@@ -316,7 +316,7 @@
<!-- dependency versions -->
<!-- change threetenbp exclusion version -->
<google.cloud.version>2.78.0</google.cloud.version>
- <aws2.version>2.59.0</aws2.version>
+ <aws2.version>2.59.1</aws2.version>
<!-- WARNING: when you upgrade asm make sure that you update the
OpCode in the initializer in
org.apache.tika.parser.asm.XHTMLClassVisitor
See TIKA-2992.
@@ -394,8 +394,7 @@
<jsr305.version>3.0.2</jsr305.version>
<!-- junit 6.x requires jdk 17 -->
<junit5.version>5.14.4</junit5.version>
- <!-- TODO 2.5.1 fails in Apache Tika text parser module -->
- <juniversalchardet.version>2.5.0</juniversalchardet.version>
+ <juniversalchardet.version>2.5.2</juniversalchardet.version>
<junrar.version>8.1.2</junrar.version>
<jwarc.version>0.37.0</jwarc.version>
<kafka.version>4.4.0</kafka.version>
@@ -426,8 +425,9 @@
<parso.version>2.0.14</parso.version>
<pax.exam.version>4.14.0</pax.exam.version>
<pdfbox.version>3.0.8</pdfbox.version>
+ <!-- 6.x requires jdk17 -->
<poi.version>5.5.1</poi.version>
- <protobuf.version>4.33.6</protobuf.version>
+ <protobuf.version>4.36.2</protobuf.version>
<quartz.version>2.5.2</quartz.version>
<reactor.core.version>3.8.8</reactor.core.version>
<reactor.netty.version>1.3.8</reactor.netty.version>
@@ -455,10 +455,10 @@
<woodstox.core.version>7.3.0</woodstox.core.version>
<xmpcore.version>6.1.11</xmpcore.version>
<!-- This has to be < 3.9 to work with Solr 8's integration tests (doesn't
fail on windows) -->
- <zookeeper.version>3.8.7</zookeeper.version>
- <zstd.version>1.5.7-26</zstd.version>
+ <zookeeper.version>3.8.8</zookeeper.version>
+ <zstd.version>1.5.7-28</zstd.version>
<nimbus-jose-jwt.version>10.10</nimbus-jose-jwt.version>
- <logback.version>1.6.5</logback.version>
+ <logback.version>1.6.6</logback.version>
<javacpp.version>1.5.14</javacpp.version>
<maven.site.version>3.22.0</maven.site.version>
<maven.exec.version>3.7.0</maven.exec.version>
@@ -1128,7 +1128,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
- <version>3.6.3</version>
+ <version>3.7.0</version>
</plugin>
</plugins>
</pluginManagement>
@@ -1153,7 +1153,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>versions-maven-plugin</artifactId>
- <version>2.25.1</version>
+ <version>2.25.2</version>
<configuration>
<generateBackupPoms>false</generateBackupPoms>
</configuration>
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/UniversalEncodingListener.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/UniversalEncodingListener.java
index 8458aabe8d..d20f88dfbb 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/UniversalEncodingListener.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/main/java/org/apache/tika/parser/txt/UniversalEncodingListener.java
@@ -56,6 +56,10 @@ class UniversalEncodingListener implements CharsetListener {
}
public void report(String name) {
+ // juniversalchardet 2.5.2 reports US-ASCII; bytes past the probe may
not be
+ if (Constants.CHARSET_US_ASCII.equals(name)) {
+ name = Constants.CHARSET_WINDOWS_1252;
+ }
if (Constants.CHARSET_WINDOWS_1252.equals(name)) {
if (hint != null) {
// Use the encoding hint when available
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/txt/TXTParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/txt/TXTParserTest.java
index f39be16c47..6a04c29426 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/txt/TXTParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-text-module/src/test/java/org/apache/tika/parser/txt/TXTParserTest.java
@@ -87,6 +87,22 @@ public class TXTParserTest extends TikaTest {
assertEquals("\n", handler.toString());
}
+ // juniversalchardet reports US-ASCII from the probe alone; the tail must
still decode
+ @Test
+ public void testAsciiPrefixLatin1Tail() throws Exception {
+ StringBuilder sb = new StringBuilder();
+ while (sb.length() < 20000) {
+ sb.append("plain ascii text\n");
+ }
+ sb.append("caf\u00e9\n");
+ Metadata metadata = new Metadata();
+ BodyContentHandler handler = new BodyContentHandler(-1);
+ parser.parse(new
ByteArrayInputStream(sb.toString().getBytes(ISO_8859_1)), handler,
+ metadata, new ParseContext());
+ assertEquals("text/plain; charset=ISO-8859-1",
metadata.get(Metadata.CONTENT_TYPE));
+ assertContains("caf\u00e9", handler.toString());
+ }
+
/**
* Test for the heuristics that we use to assign an eight-bit character
* encoding to mostly ASCII sequences. If a more specific match can not
diff --git a/tika-server/pom.xml b/tika-server/pom.xml
index e37c717f0a..58c3499ce3 100644
--- a/tika-server/pom.xml
+++ b/tika-server/pom.xml
@@ -41,7 +41,7 @@
<maven.compiler.target>17</maven.compiler.target>
<maven.compiler.release>17</maven.compiler.release>
<cxf.version>4.2.4</cxf.version>
- <jetty.version>12.1.14</jetty.version>
+ <jetty.version>12.1.15</jetty.version>
<jakarta.ws.rs.version>4.0.0</jakarta.ws.rs.version>
</properties>
diff --git a/tika-server/tika-server-core/pom.xml
b/tika-server/tika-server-core/pom.xml
index 3dc6fe5834..0a74a801ec 100644
--- a/tika-server/tika-server-core/pom.xml
+++ b/tika-server/tika-server-core/pom.xml
@@ -304,7 +304,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>{maven.exec.version}</version>
+ <version>${maven.exec.version}</version>
<executions>
<execution>
<phase>test</phase>