This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 48be18f100 TIKA-5052: drop a statistical UTF-32 pick whose bytes are
not valid code points (#3596)
48be18f100 is described below
commit 48be18f10062a09fe55aaf2ddcb1be2bebe09b5e
Author: Tim Allison <[email protected]>
AuthorDate: Mon Nov 9 16:24:57 2026 -0500
TIKA-5052: drop a statistical UTF-32 pick whose bytes are not valid code
points (#3596)
---
CHANGES.txt | 4 ++
.../ml/chardetect/MojibusterEncodingDetector.java | 12 +++--
.../tika/ml/chardetect/WideUnicodeDetector.java | 61 +++++++++++-----------
.../ml/chardetect/WideUnicodeDetectionTest.java | 28 ++++++++++
4 files changed, 72 insertions(+), 33 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index bf993b76f6..b8dab5dccd 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -412,6 +412,10 @@ Release 4.3.0 - ???
* Charset detection keeps x-MacRoman instead of relabelling it windows-1252
when the
text has MacRoman curly quotes (TIKA-5052).
+ * A UTF-32 guess from the statistical model is dropped when the bytes are
not valid
+ UTF-32 code points: binary with a few NULs in its header (a shader file
of ASCII 0/1
+ after an 80 37 00 00 header) was reported as UTF-32BE (TIKA-5052).
+
* XHTML output no longer contains empty or whitespace-only
b/strong/i/em/sup/sub
elements, and whitespace at their edges is written outside them (<b>Note:
</b>
becomes <b>Note:</b> ); u and s are unchanged. The Markdown writer does
the same for
diff --git
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
index c5fe40f2fb..c829b0197c 100644
---
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
+++
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
@@ -250,10 +250,11 @@ public class MojibusterEncodingDetector implements
EncodingDetector {
java.util.List<EncodingResult> pool = new java.util.ArrayList<>();
- // UTF-32 codepoint validity, plus UTF-16 surrogate validity for the
model's UTF-16 picks.
+ // UTF-32 codepoint validity, plus UTF-16/UTF-32 validity for the
model's wide picks.
WideUnicodeDetector.Result wide = WideUnicodeDetector.analyze(withBom);
- LOG.trace("mojibuster wideUnicode charset={} invalidLE={}
invalidBE={}",
- wide.charset, wide.invalidUtf16Le, wide.invalidUtf16Be);
+ LOG.trace("mojibuster wideUnicode charset={} invalidLE={} invalidBE={}
invalid32LE={} invalid32BE={}",
+ wide.charset, wide.invalidUtf16Le, wide.invalidUtf16Be,
+ wide.invalidUtf32Le, wide.invalidUtf32Be);
if (wide.charset != null) {
pool.add(new EncodingResult(wide.charset, UTF32_STRUCTURAL_CONF,
wide.charset.name(),
EncodingResult.ResultType.STRUCTURAL));
@@ -347,6 +348,11 @@ public class MojibusterEncodingDetector implements
EncodingDetector {
LOG.trace("mojibuster veto {} (broken surrogates)", name);
continue;
}
+ if (("UTF-32LE".equals(name) && wide.invalidUtf32Le)
+ || ("UTF-32BE".equals(name) && wide.invalidUtf32Be)) {
+ LOG.trace("mojibuster veto {} (invalid code points)", name);
+ continue;
+ }
// False-CJK veto: a legacy multi-byte CJK pick whose bytes don't
// validate (high decode-failure on the UTF-8-stripped remainder)
is
// Latin/Cyrillic/garbage mis-read as CJK. Drop it — if it was
the model's
diff --git
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/WideUnicodeDetector.java
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/WideUnicodeDetector.java
index 1fc471ba0f..cb948f30c4 100644
---
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/WideUnicodeDetector.java
+++
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/WideUnicodeDetector.java
@@ -38,7 +38,8 @@ import java.nio.charset.Charset;
* {@link Result#invalidUtf16Le} carry whether the probe contains
* structurally impossible UTF-16 surrogate sequences under each
* endianness, so callers can suppress the model's UTF-16 labels
- * when the bytes cannot be valid UTF-16. A high surrogate in the last
+ * when the bytes cannot be valid UTF-16. {@link Result#invalidUtf32Be} and
+ * {@link Result#invalidUtf32Le} do the same for the model's UTF-32 labels. A
high surrogate in the last
* code unit of the probe is a pair cut by the probe boundary, not an
error.</p>
*
* <p>All methods are stateless and safe to call from multiple threads.</p>
@@ -59,14 +60,21 @@ final class WideUnicodeDetector {
public final boolean invalidUtf16Be;
/** True if the probe contains invalid UTF-16-LE surrogate sequences.
*/
public final boolean invalidUtf16Le;
+ /** True if some 4-byte group is not a code point under UTF-32BE. */
+ public final boolean invalidUtf32Be;
+ /** True if some 4-byte group is not a code point under UTF-32LE. */
+ public final boolean invalidUtf32Le;
- private Result(Charset charset, boolean invalidUtf16Be, boolean
invalidUtf16Le) {
+ private Result(Charset charset, boolean invalidUtf16Be, boolean
invalidUtf16Le,
+ boolean invalidUtf32Be, boolean invalidUtf32Le) {
this.charset = charset;
this.invalidUtf16Be = invalidUtf16Be;
this.invalidUtf16Le = invalidUtf16Le;
+ this.invalidUtf32Be = invalidUtf32Be;
+ this.invalidUtf32Le = invalidUtf32Le;
}
- static final Result EMPTY = new Result(null, false, false);
+ static final Result EMPTY = new Result(null, false, false, false,
false);
}
/**
@@ -85,11 +93,17 @@ final class WideUnicodeDetector {
// UTF-32 must be tested before UTF-16: Latin UTF-32 also triggers the
// UTF-16 null-column check (every other stride-2 pair is (0x00,
0x00)).
- Charset utf32 = tryUtf32(bytes, offset, length);
- if (utf32 != null) {
- return new Result(utf32, false, false);
+ int[] utf32 = utf32Invalid(bytes, offset, length);
+ boolean invalidBe = utf32[0] > 0;
+ boolean invalidLe = utf32[1] > 0;
+ if (!invalidBe && utf32[2] >= UTF32_MIN_VALID) {
+ return new Result(Charset.forName("UTF-32BE"), false, false,
false, invalidLe);
}
- return tryUtf16(bytes, offset, length);
+ if (!invalidLe && utf32[2] >= UTF32_MIN_VALID) {
+ return new Result(Charset.forName("UTF-32LE"), false, false,
invalidBe, false);
+ }
+ Result utf16 = tryUtf16(bytes, offset, length);
+ return new Result(null, utf16.invalidUtf16Be, utf16.invalidUtf16Le,
invalidBe, invalidLe);
}
/** Convenience overload for a full array. */
@@ -118,8 +132,8 @@ final class WideUnicodeDetector {
private static final int UTF32_MIN_VALID = 2;
/**
- * Checks if every 4-byte group decodes to a valid Unicode codepoint
- * (U+0000–U+10FFFF, excluding surrogates U+D800–U+DFFF).
+ * Counts the 4-byte groups that are not a Unicode codepoint
+ * (U+0000–U+10FFFF, excluding surrogates U+D800–U+DFFF) under each order.
*
* <p>Inspired by ICU4J's {@code CharsetRecog_UTF_32}: the valid codepoint
* range is tiny (0x000000–0x10FFFF) relative to the 32-bit value space,
@@ -127,14 +141,12 @@ final class WideUnicodeDetector {
* immediately disqualify it. No null-byte density threshold is needed,
* which means non-BMP content (emoji, historic scripts) works
perfectly.</p>
*/
- private static Charset tryUtf32(byte[] bytes, int offset, int length) {
+ /** {invalid BE groups, invalid LE groups, groups} over the first 512
bytes. */
+ private static int[] utf32Invalid(byte[] bytes, int offset, int length) {
int sampleLen = (Math.min(length, 512) / 4) * 4;
- if (sampleLen < 8) {
- return null;
- }
int groups = sampleLen / 4;
- int validBe = 0, invalidBe = 0;
- int validLe = 0, invalidLe = 0;
+ int invalidBe = 0;
+ int invalidLe = 0;
for (int g = 0; g < groups; g++) {
int base = offset + g * 4;
@@ -144,27 +156,16 @@ final class WideUnicodeDetector {
int b3 = bytes[base + 3] & 0xFF;
int cpBe = (b0 << 24) | (b1 << 16) | (b2 << 8) | b3;
- if (cpBe >= 0 && cpBe <= 0x10FFFF && (cpBe < 0xD800 || cpBe >
0xDFFF)) {
- validBe++;
- } else {
+ if (!(cpBe >= 0 && cpBe <= 0x10FFFF && (cpBe < 0xD800 || cpBe >
0xDFFF))) {
invalidBe++;
}
int cpLe = (b3 << 24) | (b2 << 16) | (b1 << 8) | b0;
- if (cpLe >= 0 && cpLe <= 0x10FFFF && (cpLe < 0xD800 || cpLe >
0xDFFF)) {
- validLe++;
- } else {
+ if (!(cpLe >= 0 && cpLe <= 0x10FFFF && (cpLe < 0xD800 || cpLe >
0xDFFF))) {
invalidLe++;
}
}
-
- if (invalidBe == 0 && validBe >= UTF32_MIN_VALID) {
- return Charset.forName("UTF-32BE");
- }
- if (invalidLe == 0 && validLe >= UTF32_MIN_VALID) {
- return Charset.forName("UTF-32LE");
- }
- return null;
+ return new int[] {invalidBe, invalidLe, groups};
}
// -----------------------------------------------------------------------
@@ -238,7 +239,7 @@ final class WideUnicodeDetector {
}
}
- return new Result(null, invalidBe, invalidLe);
+ return new Result(null, invalidBe, invalidLe, false, false);
}
}
diff --git
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/WideUnicodeDetectionTest.java
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/WideUnicodeDetectionTest.java
index 0acf7c1b06..4c21438acb 100644
---
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/WideUnicodeDetectionTest.java
+++
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/WideUnicodeDetectionTest.java
@@ -180,6 +180,34 @@ public class WideUnicodeDetectionTest {
assertEquals(0.1f, results.get(0).getConfidence());
}
+ /**
+ * Header {@code 80 37 00 00} then ASCII '0'/'1' (a CC .mtsl2 shader
file): the model scores only the
+ * header's NULs and says UTF-32BE, but no 4-byte group is a code point.
+ */
+ @Test
+ public void bitstringWithBinaryHeaderIsNotUtf32() {
+ byte[] bytes = new byte[10240];
+ bytes[0] = (byte) 0x80;
+ bytes[1] = 0x37;
+ Random r = new Random(380107L);
+ for (int i = 4; i < bytes.length; i++) {
+ bytes[i] = (byte) (r.nextBoolean() ? '1' : '0');
+ }
+ WideUnicodeDetector.Result wide = WideUnicodeDetector.analyze(bytes);
+ assertTrue(wide.invalidUtf32Be);
+ assertTrue(wide.invalidUtf32Le);
+ assertEquals("windows-1252", top(bytes));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"UTF-32BE", "UTF-32LE"})
+ public void validUtf32HasNoInvalidFlag(String charset) {
+ WideUnicodeDetector.Result wide = WideUnicodeDetector.analyze(
+ "Grüße, мир, 世界".getBytes(Charset.forName(charset)));
+ assertEquals(charset, wide.charset.name());
+ assertFalse("UTF-32BE".equals(charset) ? wide.invalidUtf32Be :
wide.invalidUtf32Le);
+ }
+
/** UTF-16LE of {@code text} with raw code {@code units} inserted at char
index {@code at}. */
private static byte[] le(String text, int at, int... units) {
byte[] head = text.substring(0,
at).getBytes(StandardCharsets.UTF_16LE);
