This is an automated email from the ASF dual-hosted git repository.

tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git


The following commit(s) were added to refs/heads/main by this push:
     new 7bba21fae1  TIKA-5052: UTF-32 and UTF-16 structural checks read the 
whole probe (#3632)
7bba21fae1 is described below

commit 7bba21fae1c70259f3ef0da74d5199356d04a5c9
Author: Tim Allison <[email protected]>
AuthorDate: Tue Nov 17 11:21:21 2026 -0500

     TIKA-5052: UTF-32 and UTF-16 structural checks read the whole probe (#3632)
    
    utf32Invalid and the surrogate scan sampled 512 bytes, so ar symbol tables, 
zero-led
      disk images and NUL-led logs passed as UTF-32BE and decoded to 
replacement characters;
      an all-NUL prefix no longer counts as UTF-32.
    
      Co-Authored-By: Claude Fable 5.1 <[email protected]>
---
 CHANGES.txt                                        |  6 ++
 .../tika/ml/chardetect/WideUnicodeDetector.java    | 27 +++++---
 .../ml/chardetect/WideUnicodeDetectionTest.java    | 78 +++++++++++++++++++++-
 3 files changed, 100 insertions(+), 11 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 55287541b6..a8f0c3d840 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -519,6 +519,12 @@ Release 4.3.0 - ???
      UTF-32 code points: binary with a few NULs in its header (a shader file 
of ASCII 0/1
      after an 80 37 00 00 header) was reported as UTF-32BE (TIKA-5052).
 
+   * The structural UTF-32 check and the UTF-16 surrogate check read the whole 
detection
+     sample, not its first 512 bytes, and an all-NUL sample is not UTF-32: ar 
symbol
+     tables, zero-led disk images and NUL-led logs were reported as UTF-32BE 
and decoded
+     to replacement characters, and binary with a text-like head kept a UTF-16 
label
+     (TIKA-5052).
+
    * Charset detection no longer reads ASCII text with a few control 
characters (logs with
      escape or form-feed bytes, IRC colour codes, record separators, banners) 
as BOM-less
      UTF-16: when the model returns UTF-16 or UTF-32 as its only candidate, 
the junk filter
diff --git 
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/WideUnicodeDetector.java
 
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/WideUnicodeDetector.java
index cb948f30c4..fbd273d605 100644
--- 
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/WideUnicodeDetector.java
+++ 
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/WideUnicodeDetector.java
@@ -96,10 +96,11 @@ final class WideUnicodeDetector {
         int[] utf32 = utf32Invalid(bytes, offset, length);
         boolean invalidBe = utf32[0] > 0;
         boolean invalidLe = utf32[1] > 0;
-        if (!invalidBe && utf32[2] >= UTF32_MIN_VALID) {
+        // all-NUL groups are code points under both orders but evidence of 
nothing (NUL-led logs)
+        if (!invalidBe && utf32[3] >= UTF32_MIN_VALID) {
             return new Result(Charset.forName("UTF-32BE"), false, false, 
false, invalidLe);
         }
-        if (!invalidLe && utf32[2] >= UTF32_MIN_VALID) {
+        if (!invalidLe && utf32[3] >= UTF32_MIN_VALID) {
             return new Result(Charset.forName("UTF-32LE"), false, false, 
invalidBe, false);
         }
         Result utf16 = tryUtf16(bytes, offset, length);
@@ -124,7 +125,7 @@ final class WideUnicodeDetector {
     // -----------------------------------------------------------------------
 
     /**
-     * Minimum number of valid codepoints required for detection. At 8 bytes
+     * Minimum number of valid non-NUL codepoints required for detection. At 8 
bytes
      * we get exactly 2 groups; the probability that 2 random 4-byte values
      * both fall in 0x000000–0x10FFFF (excluding surrogates) is ~1.6e-11,
      * so 2 is safe against false positives on real-world byte data.
@@ -141,12 +142,17 @@ final class WideUnicodeDetector {
      * immediately disqualify it. No null-byte density threshold is needed,
      * which means non-BMP content (emoji, historic scripts) works 
perfectly.</p>
      */
-    /** {invalid BE groups, invalid LE groups, groups} over the first 512 
bytes. */
+    /**
+     * {invalid BE groups, invalid LE groups, groups, non-NUL groups} over the 
whole probe. A
+     * 512-byte sample let ar symbol tables (big-endian small ints) and 
zero-led disk images through
+     * as UTF-32BE.
+     */
     private static int[] utf32Invalid(byte[] bytes, int offset, int length) {
-        int sampleLen = (Math.min(length, 512) / 4) * 4;
+        int sampleLen = (length / 4) * 4;
         int groups = sampleLen / 4;
         int invalidBe = 0;
         int invalidLe = 0;
+        int nonNul = 0;
 
         for (int g = 0; g < groups; g++) {
             int base = offset + g * 4;
@@ -154,6 +160,9 @@ final class WideUnicodeDetector {
             int b1 = bytes[base + 1] & 0xFF;
             int b2 = bytes[base + 2] & 0xFF;
             int b3 = bytes[base + 3] & 0xFF;
+            if ((b0 | b1 | b2 | b3) != 0) {
+                nonNul++;
+            }
 
             int cpBe = (b0 << 24) | (b1 << 16) | (b2 << 8) | b3;
             if (!(cpBe >= 0 && cpBe <= 0x10FFFF && (cpBe < 0xD800 || cpBe > 
0xDFFF))) {
@@ -165,7 +174,7 @@ final class WideUnicodeDetector {
                 invalidLe++;
             }
         }
-        return new int[] {invalidBe, invalidLe, groups};
+        return new int[] {invalidBe, invalidLe, groups, nonNul};
     }
 
     // -----------------------------------------------------------------------
@@ -180,7 +189,7 @@ final class WideUnicodeDetector {
      * the main statistical model on probes that cannot be valid UTF-16.
      */
     private static Result tryUtf16(byte[] bytes, int offset, int length) {
-        int sampleLen = (Math.min(length, 512) / 2) * 2;
+        int sampleLen = (length / 2) * 2;
         if (sampleLen < 8) {
             return Result.EMPTY;
         }
@@ -225,8 +234,8 @@ final class WideUnicodeDetector {
                 }
             }
         }
-        // A high surrogate ending the sample is checked against the next 
unit; at the
-        // probe end the pair was cut, which is no evidence either way.
+        // A high surrogate ending the scan is checked against an odd trailing 
byte's unit only when one
+        // exists; at the probe end the pair was cut, which is no evidence 
either way.
         int next = offset + sampleLen;
         if (next + 1 < offset + length) {
             int even = bytes[next] & 0xFF;
diff --git 
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/WideUnicodeDetectionTest.java
 
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/WideUnicodeDetectionTest.java
index 28eb750f15..d2029cde91 100644
--- 
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/WideUnicodeDetectionTest.java
+++ 
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/WideUnicodeDetectionTest.java
@@ -18,6 +18,7 @@ package org.apache.tika.ml.chardetect;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.nio.charset.Charset;
@@ -210,6 +211,79 @@ public class WideUnicodeDetectionTest {
                 "Grüße, мир, 世界".getBytes(Charset.forName(charset)));
         assertEquals(charset, wide.charset.name());
         assertFalse("UTF-32BE".equals(charset) ? wide.invalidUtf32Be : 
wide.invalidUtf32Le);
+        // longer than any sample window, with a group cut at the probe end
+        byte[] longer = "Grüße, мир, 世界. 
".repeat(60).getBytes(Charset.forName(charset));
+        assertEquals(charset, top(Arrays.copyOf(longer, longer.length - 2)));
+    }
+
+    /**
+     * GNU ar symbol table ({@code /} member of a {@code gcc -c; ar rcs} 
archive): big-endian count and
+     * offsets, then NUL-terminated names. Every header int is a code point, 
so a check over the first
+     * bytes only called the 1.2M corpus's .a/.deb members UTF-32BE and 
decoded the names to U+FFFD.
+     */
+    @Test
+    public void arSymbolTableIsNotUtf32() {
+        int[] header = {7, 0xcc, 0xcc, 0xcc, 0xcc, 0x6e8, 0x6e8, 0x6e8};
+        String names = 
"alpha_function\0beta_function\0gamma_function\0delta_variable\0"
+                + "epsilon_function\0zeta_function\0eta_variable\0\0";
+        byte[] bytes = new byte[header.length * 4 + names.length()];
+        for (int i = 0; i < header.length; i++) {
+            bytes[i * 4 + 2] = (byte) (header[i] >>> 8);
+            bytes[i * 4 + 3] = (byte) header[i];
+        }
+        System.arraycopy(names.getBytes(StandardCharsets.US_ASCII), 0, bytes, 
header.length * 4,
+                names.length());
+        WideUnicodeDetector.Result wide = WideUnicodeDetector.analyze(bytes);
+        assertTrue(wide.invalidUtf32Be);
+        assertTrue(wide.invalidUtf32Le);
+        assertFalse(top(bytes).startsWith("UTF-32"));
+        // 128 header ints fill the old 512-byte sample on their own
+        byte[] big = new byte[128 * 4 + names.length()];
+        for (int i = 0; i < 128; i++) {
+            big[i * 4 + 2] = (byte) (i >>> 6);
+            big[i * 4 + 3] = (byte) (i * 24);
+        }
+        System.arraycopy(names.getBytes(StandardCharsets.US_ASCII), 0, big, 
512, names.length());
+        assertTrue(WideUnicodeDetector.analyze(big).invalidUtf32Be);
+        assertFalse(top(big).startsWith("UTF-32"));
+    }
+
+    /** A zero-led partition image: 512 NUL bytes, then data no 4-byte group 
of which is a code point. */
+    @Test
+    public void zeroLedImageIsNotUtf32() {
+        byte[] bytes = new byte[8192];
+        Random r = new Random(5062L);
+        for (int i = 512; i < bytes.length; i++) {
+            bytes[i] = (byte) (0x80 | r.nextInt(128));
+        }
+        WideUnicodeDetector.Result wide = WideUnicodeDetector.analyze(bytes);
+        assertTrue(wide.invalidUtf32Be);
+        assertTrue(wide.invalidUtf32Le);
+        String top = top(bytes);
+        assertFalse(top.startsWith("UTF-32") || top.startsWith("UTF-16"), top);
+    }
+
+    /** 17 KB of NUL then an ASCII log (a 1.2M MOZILLA .zip member): the 
all-NUL prefix is not UTF-32. */
+    @Test
+    public void nulLedLogIsNotUtf32() {
+        byte[] nuls = new byte[17412];
+        assertNull(WideUnicodeDetector.analyze(nuls).charset);
+        assertFalse(top(nuls).startsWith("UTF-32"), top(nuls));
+        byte[] log = "1074222328[40404160]: http request 
[\n".repeat(400).getBytes(StandardCharsets.US_ASCII);
+        byte[] bytes = Arrays.copyOf(nuls, nuls.length + log.length);
+        System.arraycopy(log, 0, bytes, nuls.length, log.length);
+        assertTrue(WideUnicodeDetector.analyze(bytes).invalidUtf32Be);
+        assertFalse(top(bytes).startsWith("UTF-32"), top(bytes));
+    }
+
+    /** A lone surrogate past byte 512 (binary with a text-like head) still 
vetoes UTF-16. */
+    @Test
+    public void loneSurrogateDeepInTheProbeIsInvalid() {
+        String prefix = "Full thread dump Java HotSpot(TM) 64-Bit Server VM 
".repeat(12);
+        assertTrue(prefix.length() > 512);
+        assertTrue(WideUnicodeDetector.analyze(le(prefix + "x", 
prefix.length(), 0xD83D)).invalidUtf16Le);
+        assertFalse(WideUnicodeDetector.analyze(le(prefix + "x", 
prefix.length(), 0xD83D, 0xDE00))
+                .invalidUtf16Le);
     }
 
     /** UTF-16LE of {@code text} with raw code {@code units} inserted at char 
index {@code at}. */
@@ -235,9 +309,9 @@ public class WideUnicodeDetectionTest {
         assertTrue(WideUnicodeDetector.analyze(le("abcdefghx", 8, 
0xD83D)).invalidUtf16Le);
     }
 
-    /** The surrogate check samples 512 bytes; a pair cut there is checked 
against the next unit. */
+    /** A pair straddling byte 512 is one pair; a lone high surrogate there is 
still invalid. */
     @Test
-    public void surrogateCutAtSampleEndLooksAhead() {
+    public void surrogateStraddlingByte512() {
         String prefix = "x".repeat(255);
         assertFalse(WideUnicodeDetector.analyze(le(prefix + " tail", 255, 
0xD83D, 0xDE00))
                 .invalidUtf16Le);

Reply via email to